refine-ruby 0.2.2

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 1308b88676521f405e1a5c11e3d3817ee42749de
+  data.tar.gz: f5c27243aaafaebb80ce2806dcea697425f57b6e
+SHA512:
+  metadata.gz: 02faba1c28168da4e55a065bc8e2aa2ba33c60dec3c077e76597e4a826e06fb216a74864fefc82ed8b43120bb5d0af8ab953a3400e29b0e460843324de7d7984
+  data.tar.gz: 2b8b9a0ac382c4372701303d5504b2190d61cb134880a0cf989b914f5c134d3c3df4e958e4f3ab3f11c196001e8d9d8f10846cc70f479315d6c423d8afe9e89d

data/.gitignore

@@ -0,0 +1,2 @@
+.DS_Store
+Gemfile.lock

data/Gemfile

@@ -0,0 +1,5 @@
+source 'https://rubygems.org'
+
+source ENV['PRIVATE_GEM_REPO'] if ENV['PRIVATE_GEM_REPO']
+# Specify your gem's dependencies in da-api.gemspec
+gemspec

data/LICENSE

@@ -0,0 +1,20 @@
+Copyright (c) 2018 David Huynh, Max Ogden & Distill Analytics Inc.
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.textile

@@ -0,0 +1,84 @@
+<h1>google-refine</h1> is a Ruby Gem client library for "OpenRefine":http://openrefine.org.  It was written for Google Refine 2.x, but should work with OpenRefine as well.
+
+If you want to port this to another language, check out the "Refine API":https://github.com/maxogden/refine-python/wiki/Refine-API documentation.
+
+NOTE: The Refine client/server protocol is an internal API which is subject to change, so use at your own risk (although it has stayed relatively stable for the last few versions).
+
+h2. Install
+
+@gem install google-refine@
+
+h2. Example
+
+Given that you have the following raw data:
+
+<pre>
+  <code>
+    Date
+    7 December 2001
+    July 1 2002
+    10/20/10
+  </code>
+</pre>
+
+Google Refine lets you clean up the data and export your operation history as a JSON instruction set. Here is an example that extracts the year from the above dates:
+
+<pre>
+<code>
+  [
+    {
+      "op": "core/text-transform",
+      "description": "Text transform on cells in column Date using expression grel:value.toDate()",
+      "engineConfig": {
+        "facets": [],
+        "mode": "row-based"
+      },
+      "columnName": "Date",
+      "expression": "grel:value.toDate()",
+      "onError": "set-to-blank",
+      "repeat": false,
+      "repeatCount": 10
+    },
+    {
+      "op": "core/text-transform",
+      "description": "Text transform on cells in column Date using expression grel:value.datePart(\"year\")+1",
+      "engineConfig": {
+        "facets": [],
+        "mode": "row-based"
+      },
+      "columnName": "Date",
+      "expression": "grel:value.datePart(\"year\")",
+      "onError": "set-to-blank",
+      "repeat": false,
+      "repeatCount": 10
+    }
+  ]
+</code>
+</pre>
+
+You can use this gem to apply the operation set to the raw data from ruby. You will need to have Google Refine running on your local computer, or specify an external address (see source):
+
+<pre>
+  <code>
+    prj = Refine.new('project_name' => 'date cleanup', 'file_name' => 'dates.txt')
+    prj.apply_operations('operations.json')
+    puts prj.export_rows('csv')
+    prj.delete_project
+  </code>
+</pre>
+
+Which outputs:
+
+<pre>
+  <code>
+    Date
+    2001
+    2002
+    2010
+  </code>
+</pre>
+
+h2. Copyright
+
+Copyright (c) 2011 David Huynh and Max Ogden. See LICENSE for details.
+

data/Rakefile

@@ -0,0 +1,8 @@
+require 'bundler/gem_tasks'
+require 'rake/testtask'
+
+Rake::TestTask.new do |t|
+  t.pattern = "test/test_*.rb"
+end
+
+task default: :test

data/VERSION

@@ -0,0 +1 @@
+0.0.1

data/bin/console

@@ -0,0 +1,11 @@
+#!/usr/bin/env ruby
+
+require "bundler/setup"
+require "refine"
+
+# You can add fixtures and/or initialization code here to make experimenting
+# with your gem easier. You can also use a different console, if you like.
+
+# (If you use this, don't forget to add pry to your Gemfile!)
+require "pry"
+Pry.start

data/lib/refine.rb

@@ -0,0 +1,231 @@
+require 'httpclient'
+require 'cgi'
+require 'json'
+require "addressable/uri"
+require 'pry'
+
+class Refine
+  attr_reader :project_name
+  attr_reader :project_id
+
+  def self.get_all_project_metadata(server="http://127.0.0.1:3333")
+    uri = "#{server}/command/core/get-all-project-metadata"
+    response = HTTPClient.new().get(uri)
+    JSON.parse(response.body)
+  end
+
+  def initialize(opts = {})
+    @server = opts["server"] || "http://127.0.0.1:3333"
+
+     @throws_exceptions = opts["throws_exceptions"] === false ? false : true
+
+    if opts["file_name"] && !opts["file_name"].empty? && opts["project_name"] && !opts["project_name"].empty?
+      project_name = CGI.escape(opts["project_name"])
+      @project_id = create_project(project_name, opts["file_name"])
+      @project_name = project_name if @project_id
+    else
+      @project_id = opts["project_id"]
+
+      metadata = self.get_project_metadata
+      @project_name = CGI.escape(metadata["name"])
+    end
+  end
+
+  def create_project(project_name, file_name)
+    uri = @server + "/command/core/create-project-from-upload"
+    project_id = false
+    File.open(file_name) do |file|
+      body = {
+        'project-file' => file,
+        'project-name' => project_name
+      }
+      response = client.post(uri, body)
+      url = response.header['Location']
+      unless url == []
+        project_id = CGI.parse(url[0].split('?')[1])['project'][0]
+      end
+    end
+    raise "Error creating project: #{response}" unless project_id
+    project_id
+  end
+
+  def apply_operations(file_name_or_string)
+    if File.exist?(file_name_or_string)
+      operations = File.read(file_name_or_string)
+    else
+      operations = file_name_or_string
+    end
+
+    call('apply-operations', 'operations' => operations)
+  end
+
+  def export_rows(opts={})
+    format = opts["format"] || 'tsv'
+    uri = @server + "/command/core/export-rows/#{@project_name}.#{format}"
+
+    body = {
+      'engine' => {
+        "facets" => opts["facets"] || [],
+        "mode" => "row-based"
+      }.to_json,
+      'options' => opts["options"] || '',
+      'project' => @project_id,
+      'format' => format
+    }
+
+    @response = client.post(uri, body)
+    @response.content
+  end
+
+  def delete_project
+    uri = @server + "/command/core/delete-project"
+    body = {
+      'project' => @project_id
+    }
+    @response = client.post(uri, body)
+    JSON.parse(@response.content)['code'] rescue false
+  end
+
+  # this pattern is pulled from mailchimp/mailchimp-gem
+
+  def call(method, params = {})
+    uri = "#{@server}/command/core/#{method}"
+    params = { "project" => @project_id }.merge(params)
+
+    response = if method.start_with?('get-')
+      client.get(uri, params)
+    else
+      client.post(uri, params)
+    end
+
+    begin
+      response = JSON.parse(response.body)
+    rescue
+      response = JSON.parse('[' + response.body + ']').first
+    end
+
+    if @throws_exceptions && response.is_a?(Hash) && response["code"] && response["code"] == "error"
+      raise "API Error: #{response}"
+    end
+
+    response
+  end
+
+  def link_to_facets(*column_names)
+    uri = Addressable::URI.parse("#{@server}/project")
+
+    facet = facet_parameters(*column_names)
+
+    json_facet=JSON::dump(facets: facet).gsub(' ', "\t")
+
+    uri.query = Addressable::URI::form_encode({project: @project_id, ui: json_facet})
+
+    uri.to_s.gsub("%09", "%20")
+
+  end
+
+    def compute_facet(*column_names)
+
+      formatted = column_names.map do |column|
+        expression, sort_by, invert = facet_opts(column.values.first)
+        {
+          "columnName" => column.keys.first,
+          "expression" => expression,
+          "name" => column.keys.first,
+          "invert" => invert,
+          "sort" => sort_by,
+          "selection" => []
+        }
+      end
+
+      json_facet = JSON::dump(facets: [formatted.first])
+
+      openrefine_response = compute_facets("engine" => json_facet)
+
+      facet_response = openrefine_response.fetch("facets").first
+
+
+
+      if facet_response.key?("choices")
+
+        choice_hash = facet_response.fetch("choices").map do |h|
+          Hash[%w(value label count selection).zip([h["v"]["v"], h["v"]["l"], h["c"], h["s"]])]
+        end
+
+        response = choice_hash.inject({}) do |hash, choice|
+          hash.merge(choice["value"] => choice["count"])
+        end
+
+      else
+      response = "Error: " + facet_response.fetch("error")
+      end
+
+    end
+
+  def facet_parameters(*column_names)
+    column_names.map do |column|
+      case column when String then
+      {
+        "c" => {
+          "columnName" => column,
+          "expression"=>"value",
+          "name"=> column,
+          "invert"=> false
+        },
+        "o" => {
+          "sort" => "name"
+        }
+      }
+      when Hash
+        expression, sort_by, invert = facet_opts(column.values.first)
+
+          {
+            "c" => {
+              "columnName" => column.keys.first,
+              "expression"=> expression,
+              "name"=> column.keys.first,
+              "invert" => invert
+            },
+            "o" => {
+              "sort" => sort_by
+            }
+          }
+      end
+    end
+  end
+
+
+
+  def method_missing(method, *args)
+    # translate: get_column_info --> get-column-info
+    call(method.to_s.gsub('_', '-'), *args)
+  end
+
+  protected
+    def facet_opts(opts_array)
+      if opts_array.is_a? String
+        expression_present = opts_array.include? "value"
+        expression = expression_present ? opts_array : "value"
+      else
+        expression_present = opts_array[0].include? "value"
+        expression = expression_present ? opts_array[0] : "value"
+      end
+
+      sort_by = opts_array.include? "sort_count"
+      invert = opts_array.include? "invert"
+
+      sort_by = sort_by ? "count" : "name"
+      invert = invert ? true : false
+
+      return escape_backticks(expression), sort_by, invert
+    end
+
+    def escape_backticks(string)
+      string.gsub('//','////')
+    end
+
+    def client
+      @client ||= HTTPClient.new()
+    end
+
+end

data/lib/refine/version.rb

@@ -0,0 +1,3 @@
+module Refine
+  VERSION = "0.2.2"
+end

data/refine-ruby.gemspec

@@ -0,0 +1,38 @@
+# -*- encoding: utf-8 -*-
+lib = File.expand_path("../lib", __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'refine/version'
+
+Gem::Specification.new do |s|
+  s.name = %q{refine-ruby}
+  s.version = Refine::VERSION
+  s.authors = ["Caleb Buxton", "Allie Dukuze", "Max Ogden", "Michael Bianco"]
+  s.summary = %q{Client library for interacting with Google Refine instances}
+  s.description = %q{Client library for interacting with Google Refine instances. Easily work with CSVs from the command line}
+  s.email = %w{caleb+refine-ruby@distillanalytics.ca allie+refine-ruby@distillanalytics.ca}
+
+  s.extra_rdoc_files = [
+    "LICENSE",
+    "README.textile"
+  ]
+
+  s.files         = `git ls-files -z`.split("\x0").reject do |f|
+    f.match(%r{^(test|spec|features)/})
+  end
+
+  s.homepage = "http://github.com/maxogden/refine-ruby"
+  s.require_paths = ["lib"]
+
+  s.test_files    = `git ls-files -z`.split("\x0").select do |f|
+    f.match(%r{^(test|spec|features)/})
+  end
+
+  s.add_dependency "addressable", '~> 2.5'
+  s.add_dependency "httpclient", "~> 2.8.3"
+  s.add_dependency "json", "~> 2.1.0"
+  s.add_development_dependency "bundler", "~> 1.10"
+  s.add_development_dependency "minitest", "~> 5.11"
+  s.add_development_dependency "pry", "~> 0.11.3"
+  s.add_development_dependency "rake", "~> 10.0"
+
+end

data/test/dates.csv

@@ -0,0 +1,4 @@
+Date
+7 December 2001
+July 1 2002
+10/20/10

data/test/dates.txt

@@ -0,0 +1,4 @@
+Date
+7 December 2001
+July 1 2002
+10/20/10

data/test/examples_of_usage.rb

@@ -0,0 +1,109 @@
+### NOTE: The internal client-server protocol used by OpenRefine is not yet maintained as a stable external API, subject to change. ###
+### Therefore, plase indicate changes you notice to kittelmann@sub.uni-goettingen.de ###
+### Some examples require cURL http://curl.haxx.se ###
+### It is assumed that examples are run from the 'test' directory. Otherwise paths need to be adjusted. 
+load '../lib/refine.rb'
+
+##########################
+### create initial project
+##########################
+prj = Refine.new({ 'project_name' => 'date_cleanup', 'file_name' => 'dates.csv' })
+
+
+##########################
+### create another project 
+##########################
+prj.create_project( 'date_cleanup', 'dates.txt' )                          # return value = project id, example: 1484090391100
+
+
+################
+### do something
+################
+prj.apply_operations( 'operations.json' )                                  # return value = status code, example: {'code'=>'ok'}
+
+
+######################
+### extract operations
+######################
+prj.get_operations                                                          # return value = operations as Hash
+
+######################################
+### save extracted operations to file:
+######################################
+extracted_operations = prj.get_operations
+File.open('../test/extracted_operations.json', 'w') do |f|
+  f.write extracted_operations
+end
+
+
+###############
+### export data
+###############
+prj.export_rows                                                              # return value = exported data as tsv 
+prj.export_rows( {'format'=>'tsv'} )                                         # return value = exported data as tsv   
+prj.export_rows( {'format'=>'csv'} )                                         # return value = exported data as csv
+
+### export data in custom table format
+prj.export_rows( { 'options'=>{'separator'=>';','lineSeparator'=>'\n'} } )   # return value = exported data as *sv with semicolon for separator
+
+### additional options available:
+prj.export_rows( { 'options'=>{'separator'=>';','lineSeparator'=>'\n', 'outputColumnHeaders'=>true, 'outputBlankRows'=>true, 'columns'=>[{'name'=>'Date1'}] } } )
+prj.export_rows( { 'options'=>{'separator'=>';','lineSeparator'=>'\n', 'outputColumnHeaders'=>false, 'outputBlankRows'=>false } } )
+
+### save extracted data to file:
+exported_data = prj.export_rows( {'format'=>'csv'} )
+File.open('../test/exported_data.csv', 'w') do |f| # works
+  f.write exported_data
+end
+
+
+##################################
+### export data using own template
+##################################
+
+### construct template as url-encoded string
+prefix = '%7B%0D%0A++%22rows%22+%3A+%5B%0D%0A'
+suffix = '%0D%0A++%5D%0D%0A%7D'
+separator = '%2C%0D%0A'
+row_template = '++++%7B%0D%0A++++++%22Column+1%22+%3A+%7B%7Bjsonize%28cells%5B%22Column+1%22%5D.value%29%7D%7D%0D%0A++++%7D'
+
+### call (using cURL http://curl.haxx.se)
+data = "engine=%7B%22facets%22%3A%5B%5D%2C%22mode%22%3A%22row-based%22%7D&project=#{prj.project_id}&format=template&sorting=%7B%22criteria%22%3A%5B%5D%7D&prefix=#{prefix}&suffix=#{suffix}&separator=#{separator}&template=#{row_template}"
+system "curl --data #{'"' + data + '"'} http://127.0.0.1:3333/command/core/export-rows/"
+
+### save extracted data to file:
+system "curl --data #{'"' + data + '"'} http://127.0.0.1:3333/command/core/export-rows/ > exported_data.json"
+
+### let Ruby do the URL encoding of the template
+prefix       = CGI.escape('{
+  "rows" : [
+')
+suffix       = CGI.escape('
+  ]
+}')
+separator    = CGI.escape(',
+')
+row_template = CGI.escape('    {
+      "Column 1" : {{jsonize(cells["Column 1"].value)}}
+    }')
+data = "engine=%7B%22facets%22%3A%5B%5D%2C%22mode%22%3A%22row-based%22%7D&project=#{prj.project_id}&format=template&sorting=%7B%22criteria%22%3A%5B%5D%7D&prefix=#{prefix}&suffix=#{suffix}&separator=#{separator}&template=#{row_template}"
+system "curl --data #{'"' + data + '"'} http://127.0.0.1:3333/command/core/export-rows/"
+
+
+#################
+### rename column
+#################
+prj.rename_column( { 'oldColumnName'=>'Date', 'newColumnName'=>'Date1' } )   # return value = status Hash, e.g. {"code"=>"ok", "historyEntry"=>{"id"=>1438598625335, "description"=>"Rename column Date to Date1", "time"=>"2015-08-03T12:29:53Z"}}
+
+
+############
+### metadata
+############
+prj.get_project_metadata                                                     # return value = metadata as Hash
+prj.get_all_project_metadata                                                 # return value = metadata for all projects as Hash
+
+
+##################
+### delete project
+##################
+prj.delete_project                                                           # return value = status, e.g. ok

data/test/operations.json

@@ -0,0 +1,28 @@
+[
+    {
+        "op": "core/text-transform",
+        "description": "Text transform on cells in column Date using expression grel:value.toDate()",
+        "engineConfig": {
+            "facets": [],
+            "mode": "row-based"
+        },
+        "columnName": "Date",
+        "expression": "grel:value.toDate()",
+        "onError": "set-to-blank",
+        "repeat": false,
+        "repeatCount": 10
+    },
+    {
+        "op": "core/text-transform",
+        "description": "Text transform on cells in column Date using expression grel:value.datePart(\"year\")",
+        "engineConfig": {
+            "facets": [],
+            "mode": "row-based"
+        },
+        "columnName": "Date",
+        "expression": "grel:value.datePart(\"year\")",
+        "onError": "set-to-blank",
+        "repeat": false,
+        "repeatCount": 10
+    }
+]

data/test/test.rb

@@ -0,0 +1,6 @@
+load '../lib/refine.rb'
+
+prj = Refine.new({ "project_name" => 'date cleanup', "file_name" => 'dates.csv' })
+prj.apply_operations('operations.json')
+puts prj.export_rows('csv')
+prj.delete_project

data/test/test_refine.rb

@@ -0,0 +1,232 @@
+gem 'minitest'
+require 'minitest/autorun'
+require_relative '../lib/refine.rb'
+
+
+describe Refine do
+
+  before do
+    @refine_project = Refine.new({ "project_name" => 'date_cleanup', "file_name" => './test/dates.txt' })
+  end
+
+  describe "error handling" do
+    it "throws an RuntimeError when @throws_exceptions == true (default)" do
+      faulty_operations = '[
+      "op": "core/text-transform",
+      "description": "Text transform on cells in column Date using expression grel:value.toDate()",
+      "engineConfig": {
+        "facets": [],
+        "mode": "row-based"
+      },
+      "onError": "set-to-blank",
+      "repeat": false,
+      "repeatCount": 10
+    }
+    ]'
+      proc {@refine_project.apply_operations(faulty_operations)}.must_raise RuntimeError
+    end
+
+    it "responds with error as a ruby hash when @throws_exceptions == false" do
+      @refine_project = Refine.new({ "project_name" => 'date_cleanup', "file_name" => './test/dates.txt', "throws_exceptions" => false })
+      faulty_operations = '[
+            "op": "core/text-transform",
+            "description": "Text transform on cells in column Date using expression grel:value.toDate()",
+            "engineConfig": {
+              "facets": [],
+              "mode": "row-based"
+            },
+            "onError": "set-to-blank",
+            "repeat": false,
+            "repeatCount": 10
+          }
+        ]'
+
+       _(@refine_project.apply_operations(faulty_operations)).must_equal (
+         {"stack"=>
+  "org.json.JSONException: Expected a ',' or ']' at 19 [character 17 line 2]\n\tat org.json.JSONTokener.syntaxError(JSONTokener.java:423)\n\tat org.json.JSONArray.<init>(JSONArray.java:143)\n\tat org.json.JSONTokener.nextValue(JSONTokener.java:356)\n\tat com.google.refine.util.ParsingUtilities.evaluateJsonStringToArray(ParsingUtilities.java:137)\n\tat com.google.refine.commands.history.ApplyOperationsCommand.doPost(ApplyOperationsCommand.java:63)\n\tat com.google.refine.RefineServlet.service(RefineServlet.java:177)\n\tat javax.servlet.http.HttpServlet.service(HttpServlet.java:820)\n\tat org.mortbay.jetty.servlet.ServletHolder.handle(ServletHolder.java:511)\n\tat org.mortbay.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1166)\n\tat org.mortbay.servlet.UserAgentFilter.doFilter(UserAgentFilter.java:81)\n\tat org.mortbay.servlet.GzipFilter.doFilter(GzipFilter.java:155)\n\tat org.mortbay.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1157)\n\tat org.mortbay.jetty.servlet.ServletHandler.handle(ServletHandler.java:388)\n\tat org.mortbay.jetty.security.SecurityHandler.handle(SecurityHandler.java:216)\n\tat org.mortbay.jetty.servlet.SessionHandler.handle(SessionHandler.java:182)\n\tat org.mortbay.jetty.handler.ContextHandler.handle(ContextHandler.java:765)\n\tat org.mortbay.jetty.webapp.WebAppContext.handle(WebAppContext.java:418)\n\tat org.mortbay.jetty.handler.HandlerWrapper.handle(HandlerWrapper.java:152)\n\tat org.mortbay.jetty.Server.handle(Server.java:326)\n\tat org.mortbay.jetty.HttpConnection.handleRequest(HttpConnection.java:542)\n\tat org.mortbay.jetty.HttpConnection$RequestHandler.content(HttpConnection.java:938)\n\tat org.mortbay.jetty.HttpParser.parseNext(HttpParser.java:755)\n\tat org.mortbay.jetty.HttpParser.parseAvailable(HttpParser.java:218)\n\tat org.mortbay.jetty.HttpConnection.handle(HttpConnection.java:404)\n\tat org.mortbay.jetty.bio.SocketConnector$Connection.run(SocketConnector.java:228)\n\tat java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)\n\tat java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)\n\tat java.lang.Thread.run(Thread.java:745)\n",
+ "code"=>"error",
+ "message"=>"Expected a ',' or ']' at 19 [character 17 line 2]"}
+       )
+    end
+
+  end
+
+  it "finding project through project id" do
+    new_refine_project = Refine.new({"project_name" => 'date_cleanup', "file_name" => './test/dates.txt'})
+
+    finding_new_refine_project_using_id = Refine.new("project_id"=> new_refine_project.project_id)
+
+    assert_equal new_refine_project.project_name, finding_new_refine_project_using_id.project_name
+
+    new_refine_project.delete_project
+  end
+
+	it "refine_initializer_has_instance_variable_project_name" do
+		assert_equal 'date_cleanup', @refine_project.project_name
+	end
+
+  it "refine_initializer_has_instance_variable_project_id" do
+    assert @refine_project.project_id.match(/^[0-9]+$/)
+  end
+
+  it "get_all_project_metadata" do
+    assert Refine.get_all_project_metadata.instance_of? Hash
+  end
+
+  it "apply_operations" do
+    assert @refine_project.apply_operations( './test/operations.json' )
+  end
+
+  it "call" do
+     assert @refine_project.call( 'apply-operations', 'operations' => File.read( './test/operations.json' ) )
+  end
+
+  describe "deep linking into a facet state" do
+
+    it "creates working url for custom expressions which include spilting a string with spaces (i.e. value.split(/[ -\/]/) )" do
+      facet_url = @refine_project.link_to_facets({"company"=>'filter(forEach(forEach(value.split(/[ -\/]/),v,v.replace(/^[^\w\s]/,"")    ),v2,v2.replace(/[^\w\s]$/,"").toLowercase()),i,isNonBlank(i))'})
+      assert_includes facet_url, "filter%28forEach%28forEach%28value.split%28%2F%5B%20-%5C%5C%2F%5D%2F%29%2Cv%2Cv.replace%28%2F%5E%5B%5E%5C%5Cw%5C%5Cs%5D%2F%2C%5C%22%5C%22%29%20%20%20%20%29%2Cv2%2Cv2.replace%28%2F%5B%5E%5C%5Cw%5C%5Cs%5D%24%2F%2C%5C%22%5C%22%29.toLowercase%28%29%29%2Ci%2CisNonBlank%28i%29%29%22%2C%22name%22%3A%22company%22%2C%22invert%22%3Afalse%7D%2C%22o%22%3A%7B%22sort%22%3A%22name%22%7D%7D%5D%7D"
+    end
+
+    it "creates working url for custom expressions which uses the equal operator '=='" do
+      facet_url = @refine_project.link_to_facets({"transcript_company"=>'filter(forEach(cells.company.value.split(/[\p{Punct}[^\s\w]]/),company_part,or(with(value.replace(/[\p{Punct}[^\s\w]]/,"").phonetic(),left,with(phonetic(company_part),right,or(left==right,or(left.contains(right),right.contains(left))))),with(value.replace(/[\p{Punct}[^\s\w]]/,"").fingerprint(),left,with(fingerprint(company_part.replace(/[\p{Punct}[^\s\w]]/,"")),right,or(left==right,or(left.contains(right),right.contains(left))))))),v,v).uniques().length()>=1'})
+      assert_includes facet_url, "filter%28forEach%28cells.company.value.split%28%2F%5B%5C%5Cp%7BPunct%7D%5B%5E%5C%5Cs%5C%5Cw%5D%5D%2F%29%2Ccompany_part%2Cor%28with%28value.replace%28%2F%5B%5C%5Cp%7BPunct%7D%5B%5E%5C%5Cs%5C%5Cw%5D%5D%2F%2C%5C%22%5C%22%29.phonetic%28%29%2Cleft%2Cwith%28phonetic%28company_part%29%2Cright%2Cor%28left%3D%3Dright%2Cor%28left.contains%28right%29%2Cright.contains%28left%29%29%29%29%29%2Cwith%28value.replace%28%2F%5B%5C%5Cp%7BPunct%7D%5B%5E%5C%5Cs%5C%5Cw%5D%5D%2F%2C%5C%22%5C%22%29.fingerprint%28%29%2Cleft%2Cwith%28fingerprint%28company_part.replace%28%2F%5B%5C%5Cp%7BPunct%7D%5B%5E%5C%5Cs%5C%5Cw%5D%5D%2F%2C%5C%22%5C%22%29%29%2Cright%2Cor%28left%3D%3Dright%2Cor%28left.contains%28right%29%2Cright.contains%28left%29%29%29%29%29%29%29%2Cv%2Cv%29.uniques%28%29.length%28%29%3E%3D1"
+    end
+
+    it "by generating urls based on a simple facet specification" do
+      # urls need the terms facet structure in json, encoded, as the UI parameter
+      raw_url = @refine_project.link_to_facets("Date")
+
+      url = URI::parse(raw_url)
+
+      assert !url.query.empty?
+
+      params = Hash[url.query.split("&").map{|i| i.split("=")}]
+
+      assert !params['ui'].nil?
+
+      facet_spec = JSON::parse(URI::decode(params['ui']))
+      assert_equal 1, facet_spec.fetch("facets").length
+
+      assert_equal "Date", facet_spec.fetch("facets").first.fetch("c").fetch("columnName")
+      assert_equal "Date", facet_spec.fetch("facets").first.fetch("c").fetch("name")
+    end
+
+    describe "depends on generating a ui query" do
+      describe "valid facet query" do
+        it "needs a control key ('c') for each facet" do
+          structure = @refine_project.facet_parameters("Date")
+          assert_includes structure.first.keys, "c"
+        end
+
+        it "each facet needs a columnName" do
+          facet = @refine_project.facet_parameters("Date").first.fetch('c')
+          assert_includes facet.keys, "columnName"
+        end
+        it "each facet needs an expression" do
+          facet = @refine_project.facet_parameters("Date").first.fetch('c')
+          assert_includes facet.keys, "expression"
+        end
+        it "each facet needs a name" do
+          facet = @refine_project.facet_parameters("Date").first.fetch('c')
+          assert_includes facet.keys, "name"
+        end
+      end
+
+      it "creates text facets for each column name provided" do
+        date_facet, company_facet = @refine_project.facet_parameters("Date","company")
+
+        assert_equal "Date", date_facet.fetch("c").fetch("columnName")
+        assert_equal "Date", date_facet.fetch("c").fetch("name")
+
+        assert_equal "company", company_facet.fetch("c").fetch("columnName")
+        assert_equal "company", company_facet.fetch("c").fetch("name")
+      end
+
+      describe "with custom expressions" do
+        it "has a `value` expression by default" do
+          date_facet, company_facet = @refine_project.facet_parameters("Date","company")
+
+          assert_equal "value", date_facet.fetch("c").fetch("expression")
+          assert_equal "value", company_facet.fetch("c").fetch("expression")
+        end
+
+        # our own expression
+        it "sanitizes custom expressions by escaping backticks" do
+          date_facet = @refine_project.link_to_facets("Date"=>'value. [\w\s] = ')
+          assert_includes date_facet, "value.%20%5B%5C%5Cw%5C%5Cs%5D%20%3D%20"
+        end
+
+        it "can pass custom expressions through a hash" do
+          date_facet, company_facet = @refine_project
+            .facet_parameters({"Date" => "value.utcTime()"},{"Company"=>"value.titleCase()"})
+
+          assert_equal "value.utcTime()", date_facet.fetch("c").fetch("expression")
+          assert_equal "Date", date_facet.fetch("c").fetch("columnName")
+
+          assert_equal "value.titleCase()", company_facet.fetch("c").fetch("expression")
+          assert_equal "Company", company_facet.fetch("c").fetch("columnName")
+        end
+      end
+
+      describe "with custom sorting" do
+        # this adds an 'o' data structure for order
+
+        it "sorts by `name` by default" do # ie existing test cases
+          date_facet = @refine_project.facet_parameters("Date")
+          assert_equal "name", date_facet.first.fetch("o").fetch("sort")
+        end
+
+        it "sorts by `count` when specified" do # need to choose a new signature / api
+          date_facet =@refine_project.facet_parameters({"Date"=>["value.utcTime()", "sort_count"]})
+          assert_equal "count", date_facet.first.fetch("o").fetch("sort")
+
+        end
+      end
+
+      describe "inversion" do
+        # find where to put these arguments in, probably 'c'
+        it "default's to not inverted" do
+          date_facet = @refine_project.facet_parameters("Date")
+          assert_equal false, date_facet.first.fetch("c").fetch("invert")
+        end
+        it "can specify to invert" do
+          date_facet = @refine_project.facet_parameters({"Date"=>["value.utcTime()", "invert"]})
+          assert_equal true, date_facet.first.fetch("c").fetch("invert")
+        end
+      end
+    end
+
+    describe "compute_facets" do
+      it "Request responds with error due to non existent column" do
+        response = @refine_project.compute_facet({"transcript_fiscal_year"=> ["isNonBlank(value)"]})
+        assert_equal("Error: No column named transcript_fiscal_year", response)
+      end
+
+      it "Request executes expression and sends response with the results" do
+        response = @refine_project.compute_facet({"Column 1"=> ["isNonBlank(value)"]})
+        assert_equal({true=>4}, response)
+      end
+
+      it "Request with faulty expression" do
+        response = @refine_project.compute_facet({"Column 1"=> ["iBlank(value)"]})
+        assert_equal("Error: Parsing error at offset 6: Unknown function or control named iBlank", response)
+      end
+    end
+
+    it "generates a link to the server" do
+      assert(URI::HTTP === URI::parse(@refine_project.link_to_facets("Date")))
+    end
+
+    # @refine_project.link_to_facets(...)
+    # => http://foo.bar/project/adfasdfsdf
+    # it "succeeds, with status 200 and a document" do
+    #   assert(HTTPClient.get(@refine_project.link_to_facets("Date")).code == 200)
+    # end
+  end
+
+  after do
+    @refine_project.delete_project
+  end
+
+end

metadata

@@ -0,0 +1,170 @@
+--- !ruby/object:Gem::Specification
+name: refine-ruby
+version: !ruby/object:Gem::Version
+  version: 0.2.2
+platform: ruby
+authors:
+- Caleb Buxton
+- Allie Dukuze
+- Max Ogden
+- Michael Bianco
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2018-05-15 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: addressable
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.5'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.5'
+- !ruby/object:Gem::Dependency
+  name: httpclient
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 2.8.3
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 2.8.3
+- !ruby/object:Gem::Dependency
+  name: json
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 2.1.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 2.1.0
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.10'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.10'
+- !ruby/object:Gem::Dependency
+  name: minitest
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '5.11'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '5.11'
+- !ruby/object:Gem::Dependency
+  name: pry
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.11.3
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.11.3
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+description: Client library for interacting with Google Refine instances. Easily work
+  with CSVs from the command line
+email:
+- caleb+refine-ruby@distillanalytics.ca
+- allie+refine-ruby@distillanalytics.ca
+executables: []
+extensions: []
+extra_rdoc_files:
+- LICENSE
+- README.textile
+files:
+- ".gitignore"
+- Gemfile
+- LICENSE
+- README.textile
+- Rakefile
+- VERSION
+- bin/console
+- lib/refine.rb
+- lib/refine/version.rb
+- refine-ruby.gemspec
+- test/dates.csv
+- test/dates.txt
+- test/examples_of_usage.rb
+- test/operations.json
+- test/test.rb
+- test/test_refine.rb
+homepage: http://github.com/maxogden/refine-ruby
+licenses: []
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.6.13
+signing_key: 
+specification_version: 4
+summary: Client library for interacting with Google Refine instances
+test_files:
+- test/dates.csv
+- test/dates.txt
+- test/examples_of_usage.rb
+- test/operations.json
+- test/test.rb
+- test/test_refine.rb