ajax-cat 0.3.0 → 0.4.0

data/Gemfile

@@ -10,6 +10,8 @@ gem 'sinatra', "~> 1.3.2"
 gem 'colorize', "~> 0.5.8"
 gem "thin", "~> 1.3.1"
 gem "thor", "~> 0.15.2"
+gem "sqlite3-ruby", "~> 1.3.3"
+gem "activerecord"
 
 group :development do
   gem "rdoc", "~> 3.12"

data/Gemfile.lock

@@ -1,17 +1,32 @@
 GEM
   remote: http://rubygems.org/
   specs:
+    activemodel (3.2.3)
+      activesupport (= 3.2.3)
+      builder (~> 3.0.0)
+    activerecord (3.2.3)
+      activemodel (= 3.2.3)
+      activesupport (= 3.2.3)
+      arel (~> 3.0.2)
+      tzinfo (~> 0.3.29)
+    activesupport (3.2.3)
+      i18n (~> 0.6)
+      multi_json (~> 1.0)
+    arel (3.0.2)
+    builder (3.0.0)
     colorize (0.5.8)
     curb (0.8.0)
     daemons (1.1.8)
     eventmachine (0.12.10)
     git (1.2.5)
+    i18n (0.6.0)
     jeweler (1.8.3)
       bundler (~> 1.0)
       git (>= 1.2.5)
       rake
       rdoc
     json (1.6.6)
+    multi_json (1.3.5)
     rack (1.4.1)
     rack-protection (1.2.0)
       rack
@@ -27,17 +42,22 @@ GEM
       rack (~> 1.3, >= 1.3.6)
       rack-protection (~> 1.2)
       tilt (~> 1.3, >= 1.3.3)
+    sqlite3 (1.3.6)
+    sqlite3-ruby (1.3.3)
+      sqlite3 (>= 1.3.3)
     thin (1.3.1)
       daemons (>= 1.0.9)
       eventmachine (>= 0.12.6)
       rack (>= 1.0.0)
     thor (0.15.2)
     tilt (1.3.3)
+    tzinfo (0.3.33)
 
 PLATFORMS
   ruby
 
 DEPENDENCIES
+  activerecord
   bundler (~> 1.0.0)
   colorize (~> 0.5.8)
   curb (~> 0.8.0)
@@ -45,5 +65,6 @@ DEPENDENCIES
   rdoc (~> 3.12)
   shoulda (~> 3.0.1)
   sinatra (~> 1.3.2)
+  sqlite3-ruby (~> 1.3.3)
   thin (~> 1.3.1)
   thor (~> 0.15.2)

data/Rakefile

@@ -24,15 +24,21 @@ end
 Jeweler::RubygemsDotOrgTasks.new
 
 require 'rake/testtask'
+Rake::TestTask.new(:test) do |test|
+  test.libs << 'lib' << 'test'
+  test.test_files = FileList['test/unit/test_*.rb','test/integration/test_*.rb']
+  test.verbose = false
+end
+
 namespace :test do
   Rake::TestTask.new(:unit) do |test|
     test.libs << 'lib' << 'test'
-    test.test_files = FileList['test/unit/test_*.rb','test/integration/*.rb']
+    test.test_files = FileList['test/unit/test_*.rb']
     test.verbose = false
   end
   Rake::TestTask.new(:integration) do |test|
     test.libs << 'lib' << 'test'
-    test.test_files = FileList['test/integration/*.rb']
+    test.test_files = FileList['test/integration/test_*.rb']
     test.verbose = false
   end
 end

data/VERSION

@@ -1 +1 @@
-0.3.0
+0.4.0

data/ajax-cat.gemspec

@@ -5,11 +5,11 @@
 
 Gem::Specification.new do |s|
   s.name = "ajax-cat"
-  s.version = "0.3.0"
+  s.version = "0.4.0"
 
   s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
   s.authors = ["Ondrej Odchazel"]
-  s.date = "2012-05-16"
+  s.date = "2012-05-22"
   s.description = "computer-aided translation backed by machine translation"
   s.email = "odchazel@gmail.com"
   s.executables = ["ajax-cat"]
@@ -30,8 +30,10 @@ Gem::Specification.new do |s|
     "lib/.DS_Store",
     "lib/ajax-cat.rb",
     "lib/ajax-cat/ajax_cat_server.rb",
+    "lib/ajax-cat/log.rb",
     "lib/ajax-cat/logger.rb",
     "lib/ajax-cat/moses_pair.rb",
+    "lib/ajax-cat/pairs.rb",
     "lib/ajax-cat/public/AjaxCatList.coffee",
     "lib/ajax-cat/public/AjaxCatTranslation.coffee",
     "lib/ajax-cat/public/Suggestions.coffee",
@@ -49,8 +51,16 @@ Gem::Specification.new do |s|
     "lib/ajax-cat/public/translation.js",
     "lib/ajax-cat/request/raw.rb",
     "lib/ajax-cat/request/simple.rb",
+    "lib/ajax-cat/request/suggestion.rb",
+    "lib/ajax-cat/request/table.rb",
+    "lib/ajax-cat/task.rb",
+    "lib/ajax-cat/views/admin.erb",
+    "lib/ajax-cat/views/experiment.erb",
+    "logs.db",
+    "test/fixtures/ajax-cat.ini.json",
     "test/fixtures/europarl.srilm.gz",
     "test/fixtures/moses.ini",
+    "test/fixtures/nbest.txt",
     "test/fixtures/phrase-table",
     "test/fixtures/translation_table",
     "test/helper.rb",
@@ -58,8 +68,11 @@ Gem::Specification.new do |s|
     "test/unit/.DS_Store",
     "test/unit/test_logger.rb",
     "test/unit/test_moses_pair.rb",
+    "test/unit/test_pairs.rb",
     "test/unit/test_raw_request.rb",
-    "test/unit/test_simple_request.rb"
+    "test/unit/test_simple_request.rb",
+    "test/unit/test_suggestion_request.rb",
+    "test/unit/test_table_request.rb"
   ]
   s.homepage = "http://github.com/hypertornado/ajax-cat"
   s.licenses = ["MIT"]
@@ -75,6 +88,8 @@ Gem::Specification.new do |s|
       s.add_runtime_dependency(%q<colorize>, ["~> 0.5.8"])
       s.add_runtime_dependency(%q<thin>, ["~> 1.3.1"])
       s.add_runtime_dependency(%q<thor>, ["~> 0.15.2"])
+      s.add_runtime_dependency(%q<sqlite3-ruby>, ["~> 1.3.3"])
+      s.add_runtime_dependency(%q<activerecord>, [">= 0"])
       s.add_development_dependency(%q<rdoc>, ["~> 3.12"])
       s.add_development_dependency(%q<bundler>, ["~> 1.0.0"])
       s.add_development_dependency(%q<jeweler>, ["~> 1.8.3"])
@@ -85,6 +100,8 @@ Gem::Specification.new do |s|
       s.add_dependency(%q<colorize>, ["~> 0.5.8"])
       s.add_dependency(%q<thin>, ["~> 1.3.1"])
       s.add_dependency(%q<thor>, ["~> 0.15.2"])
+      s.add_dependency(%q<sqlite3-ruby>, ["~> 1.3.3"])
+      s.add_dependency(%q<activerecord>, [">= 0"])
       s.add_dependency(%q<rdoc>, ["~> 3.12"])
       s.add_dependency(%q<bundler>, ["~> 1.0.0"])
       s.add_dependency(%q<jeweler>, ["~> 1.8.3"])
@@ -96,6 +113,8 @@ Gem::Specification.new do |s|
     s.add_dependency(%q<colorize>, ["~> 0.5.8"])
     s.add_dependency(%q<thin>, ["~> 1.3.1"])
     s.add_dependency(%q<thor>, ["~> 0.15.2"])
+    s.add_dependency(%q<sqlite3-ruby>, ["~> 1.3.3"])
+    s.add_dependency(%q<activerecord>, [">= 0"])
     s.add_dependency(%q<rdoc>, ["~> 3.12"])
     s.add_dependency(%q<bundler>, ["~> 1.0.0"])
     s.add_dependency(%q<jeweler>, ["~> 1.8.3"])

data/bin/ajax-cat

@@ -4,11 +4,26 @@ require_relative('../lib/ajax-cat.rb')
 
 case ARGV.first
 when "start"
-  AjaxCat::Starter.start(8585, true)
-when "stop"
-  puts "Stopping ajax-cat server."
-  AjaxCat::Starter.stop
-  puts "Server stopped."
+  AjaxCat::Runner.new(8585)
+when "-v", "--version"
+  puts VERSION
+  exit 0
+when "-h", "--help"
+	puts "no help yet, look at https://github.com/hypertornado/ajax-cat"
+	exit 0
+when "example"
+	begin
+		example = AjaxCat::Runner.example
+		f = File.open("ajax-cat.ini.json", "w")
+		f.write(example)
+		f.close
+	rescue
+		puts "Can't create example ajax-cat configuration file in #{Dir.pwd}".red
+		exit 1
+	end
+	puts "example ajax-cat configuration file created at #{Dir.pwd+'/ajax-cat.ini.json'}".green
+	exit 0
 else
   puts "Unknown command."
+  exit 0
 end

data/lib/ajax-cat.rb

@@ -5,34 +5,121 @@ require 'thin'
 require 'json'
 require 'thread'
 require 'colorize'
+require 'sqlite3'
+require 'active_record'
 require_relative 'ajax-cat/moses_pair.rb'
+require_relative 'ajax-cat/pairs.rb'
 require_relative 'ajax-cat/request/raw.rb'
 require_relative 'ajax-cat/request/simple.rb'
+require_relative 'ajax-cat/request/table.rb'
+require_relative 'ajax-cat/request/suggestion.rb'
 require_relative 'ajax-cat/ajax_cat_server.rb'
 require_relative 'ajax-cat/logger.rb'
+require_relative 'ajax-cat/log.rb'
+require_relative 'ajax-cat/task.rb'
 
-module AjaxCat
+VERSION = File.read(File.dirname(__FILE__) + "/../VERSION")
+DEFAULT_PORT = 8585
 
-  class Starter
+module AjaxCat
 
-    def self.pair
-      @@pair
+  class Runner
+    def self.pairs
+      @@pairs
     end
 
-  	def self.start(port = 8585, silent = false)
+  	def initialize(port = DEFAULT_PORT, silent = true)
       logger = Logger.new
       logger.log "starting ajax-cat"
       Thin::Logging.silent = silent
-      @@pair = MosesPair.new('cs-en',"moses","/Users/ondrejodchazel/projects/ajax-cat/ajax-cat-new/test/fixtures/moses.ini")
+      @settings = JSON.parse(File.read("ajax-cat.ini.json"))
+      port = @settings["port"] if port == DEFAULT_PORT
+      @@pairs = Pairs.new(@settings)
       Dir.chdir(Dir.home)
       Dir.mkdir('.ajax-cat') unless Dir.exist?('.ajax-cat')
       builder = Rack::Builder.new do
         map('/'){ run AjaxCatServer }
       end
       logger.log "starting server on port #{port}"
+      @server = builder
       Rack::Handler::Thin.run builder, :Port => port
-      logger.log "shutting down ajax-cat server"
+      logger.log "ajax-cat server shutted down"
+    end
+
+    def self.example
+      ret = <<-END.gsub(/^ {8}/, '')
+        {
+          "port": #{DEFAULT_PORT},
+          "pairs": [
+            {
+              "name": "de-en",
+              "moses_path": "moses",
+              "moses_ini_path": "#{File.dirname(__FILE__) + "/../test/fixtures/moses.ini"}"
+            }
+          ]
+        }
+      END
+      ret
     end
+
+    def self.create_sample_moses_ini
+      path = File.dirname(__FILE__) + "/../test/fixtures/"
+      @moses_ini = <<-END.gsub(/^ {6}/, '')
+      #########################
+      ### MOSES CONFIG FILE ###
+      #########################
+
+      # input factors
+      [input-factors]
+      0
+
+      # mapping steps, either (T) translation or (G) generation
+      [mapping]
+      T 0
+
+      # translation tables: source-factors, target-factors, number of scores, file
+      [ttable-file]
+      0 0 0 1 #{path}phrase-table
+
+      # language models: type(srilm/irstlm/kenlm), factors, order, file
+      [lmodel-file]
+      8 0 3 #{path}europarl.srilm.gz
+
+      # limit on how many phrase translations e for each phrase f are loaded
+      [ttable-limit]
+      10
+
+      # distortion (reordering) weight
+      [weight-d]
+      1
+
+      # language model weights
+      [weight-l]
+      1
+
+      # translation model weights
+      [weight-t]
+      1
+
+      # word penalty
+      [weight-w]
+      0
+
+      [n-best-list]
+      nbest.txt
+      100
+
+
+
+      END
+      f = File.new(File.dirname(__FILE__) + "/../test/fixtures/moses.ini", "w")
+      f.write(@moses_ini)
+      f.close
+
+
+    end
+
   end
 
-end
+end
+

data/lib/ajax-cat/ajax_cat_server.rb

@@ -3,17 +3,85 @@ module AjaxCat
 
 	class AjaxCatServer < Sinatra::Base
 
-		configure :production, :development do
-    	disable :logging
-  	end
-
 		get '/' do 
-			"hello world"		
+			redirect "/index.html"
 		end
 
 		get '/api/raw' do
 			req = Request::Raw.new(params[:q])
-			Starter.pair.process_request(req)
+			halt 404 unless Runner.pairs.list.member?(params[:pair])
+			Runner.pairs.process_request(req, params[:pair])
+		end
+
+		get '/api/table' do
+			req = Request::Table.new(params[:q])
+			halt 404 unless Runner.pairs.list.member?(params[:pair])
+			Runner.pairs.process_request(req, params[:pair]).to_json
+		end
+
+		get '/api/suggestion' do
+			req = Request::Suggestion.new(params[:q], params[:covered], params[:translated])
+			halt 404 unless Runner.pairs.list.member?(params[:pair])
+			Runner.pairs.process_request(req, params[:pair]).to_json
+		end
+
+		get '/api/info' do
+			{
+				"pairs" => Runner.pairs.list
+			}.to_json
+		end
+
+		get '/admin' do
+			@pairs = Runner.pairs.list
+			erb :admin, locals: {pairs: @pairs}
+		end
+
+		post '/admin/add_task' do
+			Task.new(pair: params[:pair], sentence: params[:sentence]).save
+			redirect '/admin'
+		end
+
+		post '/admin/remove_task' do
+			Task.find(params[:id]).destroy
+			redirect '/admin'
+		end
+
+		get '/admin/get_experiment' do
+			email = params["email"]
+			pair = params["pair"]
+
+			tasks = Task.where(pair: pair)
+      tasks = tasks.map {|el| el.id}
+      completed = Log.where(email: email)
+      completed = completed.map {|el| el.task_id}
+
+      pool = tasks - completed
+
+      #puts "tasks: #{tasks.inspect}, completed: #{completed.inspect}, pool: #{pool.inspect}"
+
+			@task = Task.find(pool.first)
+			{
+				task_id: @task.id,
+				pair: @task.pair,
+				sentence: @task.sentence,
+				email: email
+			}.to_json
+		end
+
+		post '/admin/save_experiment' do
+			data = JSON.parse(params[:log])
+			@task = Task.find(data["task_id"])
+			@log = Log.new()
+			@log.task_id = @task.id
+			@log.email = data["email"]
+			@log.time = Time.now
+			@log.log = data.to_json
+			@log.save
+		end
+
+		get '/experiment/:id' do
+			@task = Task.find(params[:id])
+			erb :experiment, locals: {task: @task}
 		end
 
 	end

data/lib/ajax-cat/log.rb

@@ -0,0 +1,6 @@
+
+
+class Log < ActiveRecord::Base
+	belongs_to :task
+
+end

data/lib/ajax-cat/moses_pair.rb

@@ -7,11 +7,10 @@ module AjaxCat
       @request_queue = []
       @name = name
       @queue_lock = Mutex.new
-      Dir.chdir(Dir.home + "/.ajax-cat")
   		@fifo_path = "#{Dir.home}/.ajax-cat/#{name}_fifo.fifo"
-      system("rm #{@fifo_path}; mkfifo #{@fifo_path}")
+      system("rm -f #{@fifo_path}; mkfifo #{@fifo_path}")
       t1 = Thread.new{reader()}
-      @pipe = IO.popen("#{moses_path} -f #{moses_ini_path} -n-best-list - 300 distinct > #{@fifo_path} 2>/dev/null", "w")
+      @pipe = IO.popen("#{moses_path} -f #{moses_ini_path} -n-best-list - 300 distinct -include-alignment-in-n-best true -continue-partial-translation true > #{@fifo_path} 2>/dev/null", "w")
       process_request(Request::Raw.new("start_test"))
       @logger.log "pair started".green
     end
@@ -26,7 +25,7 @@ module AjaxCat
       @queue_lock.synchronize do
         request.lock.lock
         @request_queue.push(request)
-        process_string(request.sentence)
+        process_string(request.prepare_moses_request)
       end
       #TODO: avoid active waiting somehow
       until request.processed