google_refine 0.1

data/bin/upload-to-refine

@@ -0,0 +1,43 @@
+#!/usr/bin/env ruby
+
+require File.expand_path('../../lib/google_refine', __FILE__)
+
+require 'trollop'
+require 'tempfile'
+
+opts = Trollop::options do
+  opt :host, "Address of Google Refine", :default => '0.0.0.0:3333'
+end
+
+
+Signal.trap("INT") { puts "Terminating: #{$$}" }
+
+host = opts[:host]
+
+file = nil
+
+if ARGV.size > 0 
+  file = ARGV.shift
+  abort "File \"#{file}\" does not exists." if not File.exists?(file)
+end
+
+tempfile = nil
+
+if file.nil?
+  if STDIN.tty?
+    abort "The STDIN should not be a TTY."
+  else
+    tempfile = Tempfile.new('upload-to-pipelines')
+    tempfile.close
+    
+    IO.copy_stream(STDIN, tempfile)
+    file = tempfile.path
+  end
+end
+
+
+refine = Refine.new(host)
+
+puts refine.create_project(file, :projectName => file)
+
+tempfile.unlink if not tempfile.nil?

data/lib/google_refine.rb

@@ -0,0 +1,105 @@
+require 'rest_client'
+require 'json'
+
+class Project
+
+  attr_accessor :id  
+  attr_accessor :refine
+
+  def initialize(refine, id)
+    self.refine = refine
+    self.id = id
+  end
+
+  def to_s
+    "#{self.refine.url}/project?project=#{self.id}"
+  end
+end
+
+class Job
+
+  attr_accessor :id  
+  attr_accessor :refine
+  
+  def initialize(refine, id)
+    self.refine = refine
+    self.id = id
+  end
+
+  def load_raw_data(filename)
+    RestClient.post("#{self.refine.url}/command/core/importing-controller?controller=core%2Fdefault-importing-controller&jobID=#{self.id}&subCommand=load-raw-data", :upload => File.new(filename, "rb"))
+
+    while true
+      sleep 2
+      status = RestClient.post("#{self.refine.url}/command/core/get-importing-job-status?jobID=#{self.id}", nil)
+      break if JSON[status]["job"]["config"]["state"] == "ready"
+    end
+  end
+
+  def create_project(options)
+    RestClient.post("#{self.refine.url}/command/core/importing-controller?controller=core%2Fdefault-importing-controller&jobID=#{self.id}&subCommand=create-project",
+      :format => "text/line-based/*sv",
+      :options => options.to_json)
+
+    project_id = nil
+    while project_id.nil?
+      sleep 2
+      response = RestClient.post("#{self.refine.url}/command/core/get-importing-job-status?jobID=#{self.id}", nil)
+      project_id = JSON[response]["job"]["config"]["projectID"]
+    end
+
+    Project.new(self.refine, project_id)
+  end
+
+  def cancel
+    RestClient.post("#{self.refine.url}/command/core/cancel-importing-job?jobID=#{self.id}", nil)
+  end
+
+end
+
+class Refine
+  attr_accessor :url
+
+  def initialize(url)
+    self.url = url
+  end
+  
+  def url=(url)
+    if url !~ /^http/
+      url = "http://#{url}"
+    end
+    @url = url
+  end
+
+  def create_importing_job
+    response = RestClient.post("#{self.url}/command/core/create-importing-job", nil)
+    job_id = JSON[response]["jobID"]
+    Job.new(self, job_id)
+  end
+
+  def create_project(filename, param = {})
+
+    options = {}
+    options[:format]                 = param[:format]
+    options[:projectName]            = param[:name]                      || "File \"#{filename}\" uploaded on #{Time.now}"
+    options[:encoding]               = param[:encoding]                  || ""
+    options[:separator]              = param[:separator]                 || "\\t"
+    options[:ignoreLines]            = param[:ignoreLines]               || -1
+    options[:headerLines]            = param[:headerLines]               || 0
+    options[:skipDataLines]          = param[:skipDataLines]             || 0
+    options[:limit]                  = param[:limit]                     || 1_000_000
+    options[:storeBlankRows]         = param[:storeBlankRows]            || true
+    options[:guessCellValueTypes]    = param[:guessCellValueTypes]       || true
+    options[:processQuotes]          = param[:processQuotes]             || false
+    options[:storeBlankCellsAsNulls] = param[:storeBlankCellsAsNulls]    || true
+    options[:includeFileSources]     = param[:includeFileSources]        || false
+    
+    job = create_importing_job
+    job.load_raw_data(filename)
+    project = job.create_project(options)
+    project
+    ensure
+      job.cancel if job
+  end
+
+end

metadata

@@ -0,0 +1,79 @@
+--- !ruby/object:Gem::Specification
+name: google_refine
+version: !ruby/object:Gem::Version
+  version: '0.1'
+  prerelease: 
+platform: ruby
+authors:
+- Cheng Guang-Nan
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2012-07-01 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: rest-client
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: trollop
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+description: 
+email: guangnan@chengguangnan.com
+executables:
+- upload-to-refine
+extensions: []
+extra_rdoc_files: []
+files:
+- lib/google_refine.rb
+- bin/upload-to-refine
+homepage: https://github.com/guangnan/google_refine
+licenses: []
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 1.8.23
+signing_key: 
+specification_version: 3
+summary: Upload files to refine using command line or programmtically.
+test_files: []