lumix 0.0.2-java

data/lib/lumix/proto/lookup.rb

@@ -0,0 +1,105 @@
+require 'msgpack'
+
+module Lumix
+
+  TEXT_ID = 0
+  S_BEGIN = 1
+  S_END = 2
+  T_BEGIN = 3
+  T_END = 4
+
+  class Lookup
+    def initialize
+      puts "Lookup"
+      @tags = {} # tag => token_id[]
+      @words = {} # word => token_id[]
+      @tokens = [] # :text_id, :s_begin, :s_end, :t_begin, :t_end
+      @texts = [] # text_id
+      Signal.trap('INT'){exit}
+      at_exit do
+        save
+      end
+      load
+    end
+
+    def load
+      @dirty = false
+      return unless File.exists?('lookup.dat')
+      puts "Loading"
+      load_file :tags
+      load_file :words
+      load_file :texts
+      load_file :tokens
+    end
+
+    def save
+      return unless @dirty
+      @saving = true
+      puts "Saving"
+      save_file :tags
+      save_file :words
+      save_file :texts
+      save_file :tokens
+      @saving = false
+    end
+
+    def with(*types)
+      args = types.flatten.map{|name| instance_variable_get("@#{name}") || instance_variable_get("@#{name}",load_file(name)) }
+      yield *args
+    end
+
+    def save_file(name)
+      data = instance_variable_get("@#{name}")
+      File.open(name.to_s + '.dat', 'w') do |f|
+        f.print MessagePack.pack(data)
+      end
+    end
+
+    def load_file(name)
+      MessagePack.unpack(File.read(name.to_s + '.dat'))
+    end
+
+    def process(text_id)
+      return if @saving
+      @dirty = true
+      return true if @texts.member?(text_id)
+      @texts << text_id
+
+      yield if block_given?
+    end
+
+    def add_token(text_id, word, tag, s_begin, s_end, t_begin, t_end)
+      return if @saving
+      @dirty = true
+      id = (@tokens << [text_id, s_begin, s_end, t_begin, t_end]).size - 1
+      (@words[word] ||= []) << id
+      (@tags[tag] ||= []) << id
+    end
+
+    def find_word(re)
+      find_ids @words, re
+    end
+
+    def find_tag(re)
+      find_ids @tags, re
+    end
+
+    # returns the start indices of matching sequences
+    def union(*id_sets)
+      unified = id_sets.each_with_index.map{|c,i| c.map{|e| e-i}}
+      unified.inject(:&)
+    end
+
+    def text_range(t_begin, t_end)
+      a, b = @tokens[t_begin], @tokens[t_end]
+      return nil unless a[TEXT_ID] == b[TEXT_ID]
+      return a[TEXT_ID], a[S_BEGIN], b[S_END], a[T_BEGIN], b[T_END]
+    end
+
+    private
+    def find_ids(arr, re)
+      elems = arr.keys.grep(re)
+      elems.map{|e| arr[e]}.flatten
+    end
+  end
+end

data/lib/lumix/proto/lookup_filter.rb

@@ -0,0 +1,40 @@
+module Lumix
+  class LookupFilter
+
+    attr_reader :results, :filter
+    
+    def initialize(filter, &result_proc)
+      @filter = filter
+      @result_proc = result_proc
+
+      @re = create_re(filter)
+      @results = 0
+    end
+
+    def <<(result)
+      @results += 1
+      @result_proc[*result] if @result_proc
+    end
+
+    def apply(lookup, &block)
+      results = @re.map do |(type, re)|
+        lookup.send("find_#{type}", re)
+      end
+      lookup.union(*results).each do |id|
+        range = lookup.text_range(id, id + @re.size - 1) # TODO make more dynamic
+        block[*range] if block and range
+      end
+    end
+
+    def create_re(filter)
+      filter.scan(/(?:(?:\"([^\"]+)\")|(\S+))+/).map do |word, tag|
+        word ? [:word, to_re(word)] : [:tag, to_re(tag)]
+      end
+    end
+
+    def to_re(txt)
+      Regexp.new('^' + txt.gsub(/\s/, '_').gsub(/\*/, '\S*').gsub(/\?/, '\S') + '$')
+    end
+
+  end
+end

data/lib/lumix/proto/lookup_search.rb

@@ -0,0 +1,81 @@
+require 'lumix/lookup_filter'
+require 'lumix/text_snippet'
+require 'lumix/lookup'
+
+module Lumix
+
+  class LookupSearch
+
+    TAGGED = /([^\s\|]+)\|(\S+)/m        # Xxx|YYY
+
+    def initialize(db, progress)
+      @lookup = Lookup.new
+      @progress = progress
+    end
+
+    def concurrent_link?
+      false
+    end
+
+    def link_text(id)
+      ds = TaggedText[id]
+      @lookup.process id do
+        file, text, tagged = ds.filename, ds.text, ds.tagged
+
+        puts "Linking text #{file}"
+
+        txt_pos = 0
+        tagged.scan(TAGGED) do |word, tag|
+          tagged_begin = $~.begin(0)
+          tagged_end = $~.end(0)
+
+          # expand "x_y_z" notation to "x y z"
+          word_re = Regexp.new(Regexp.escape(word).gsub(/_/, '\s*'))
+          src_match = text[txt_pos..-1].match(word_re) # find the word
+          if src_match
+            offset = src_match.begin(0)
+            src_begin = txt_pos + offset
+            src_end = txt_pos + src_match.end(0)
+            txt_pos = src_end
+
+            @lookup.add_token(id, word, tag, src_begin, src_end, tagged_begin, tagged_end)
+          else
+            STDERR.puts "Could not find match for '#{word}' in text #{file}"
+            STDERR.puts text[(txt_pos-10)..(txt_pos+word.size+10)]
+            `echo '#{file}:#{txt_pos}:#{tagged_begin} unmatched "#{word}"' >> unlinked.lst`
+            return nil
+          end
+        end
+        return true
+      end
+    rescue => e # TODO remove this crap
+      STDERR.puts e
+      STDERR.puts e.backtrace
+      @lookup.save
+      raise e
+    end
+
+    def create_filter(f, &block)
+      Lumix::LookupFilter.new(f, &block)
+    end
+
+    def find(*filters, &block)
+      last_id = -1
+      t = nil
+      filters.flatten.each do |f|
+        f.apply(@lookup) do |text_id, s_begin, s_end, t_begin, t_end|
+          t = TaggedText[text_id] if text_id != last_id
+          last_id = text_id
+          
+          fname = File.basename(t.filename)
+          text_snippet = Lumix::TextSnippet.new(fname, t.text, s_begin, s_end)
+          tagged_snippet = Lumix::TextSnippet.new(fname, t.tagged, t_begin, t_end)
+          f << [text_snippet, tagged_snippet]
+        end
+      end
+    end
+
+  end
+
+  SearchStrategy = LookupSearch
+end

data/lib/lumix/result_view.rb

@@ -0,0 +1,93 @@
+class Java::OrgEclipseSwtWidgets::Table
+
+  attr_accessor :data, :tooltips
+  
+  def sweeten(app, opts={}, &block)
+    @data = []
+    @tooltips = []
+    super
+    @redraw_thread = Thread.new do
+      while !isDisposed
+        if @dirty
+          @dirty = false
+          perform do
+            setItemCount data.size
+            clearAll if clear_all
+          end
+        end
+        sleep 1 # TODO find a better alternative
+      end
+    end
+
+    # TODO implement tooltips
+
+    addListener swt::SetData do |e|
+      item = e.item
+      index = indexOf(item)
+      item.setText(Array(data[index]).to_java(:string))
+    end
+
+    addListener swt::Resize do |e|
+      default_weight = 1.0 / columns.size
+      current_width = @old_width
+      w = width
+      columns[0..-2].each do |c|
+        weight = c.width == 0 ? default_weight : c.width.to_f / current_width
+        c.width = w * weight
+      end
+      columns[columns.size - 1].pack
+      @old_width = w
+    end
+  end
+
+  def columns=(*titles)
+    if titles
+      titles.each do |title|
+        col = widgets::TableColumn.new(self, swt::CENTER)
+        col.setText title
+      end
+
+      setHeaderVisible true
+      setLinesVisible true
+    end
+  end
+
+  def sort=(sort)
+    sort = Hash.new(true) if [true, :all].member?(sort)
+    if sort
+      columns.each_with_index do |col, index|
+        if sort[col.text]
+          col.addListener swt::Selection do
+            if data
+              @data = data.sort_by {|e| e[index] }
+              update :clear
+            end
+          end
+        end
+      end
+    end
+  end
+end
+
+Sweet::WIDGET_DEFAULTS[:table] = {
+  :style => [:border, :virtual, :check]
+}
+Sweet::WIDGET_HACKS[Java::OrgEclipseSwtWidgets::Table] = {
+  :block_handler => :set_data,
+  :custom_code => proc {
+    def update(clear_all = false)
+      return if isDisposed
+      setItemCount data.size
+      clearAll if clear_all
+    end
+
+    def add_hit(*args)
+      opts = args.last === Hash ? args.pop : {}
+      d = opts[:data] || args
+      t = opts[:tooltips] || d
+      data << d
+      tooltips << t
+      @dirty = true
+    end
+  }
+}

data/lib/lumix/schema/001_create_tables.rb

@@ -0,0 +1,35 @@
+class CreateTables < Sequel::Migration
+
+  def up
+    create_table :texts do
+      primary_key :id
+      String :digest
+      String :text
+      String :tagged
+      String :filename
+      String :tagged_filename
+
+      index :digest
+    end
+
+    create_table :assoc do
+      primary_key :id
+      Integer :text_id, :references => :texts
+      Integer :position
+      Integer :src_begin
+      Integer :src_end
+      Integer :tagged_begin
+      Integer :tagged_end
+
+      index [:text_id, :tagged_end]
+      index [:text_id, :tagged_begin]
+      index [:text_id, :position]
+    end
+  end
+
+  def down
+    drop_table :assoc
+    drop_table :texts
+  end
+
+end

data/lib/lumix/schema/002_categories.rb

@@ -0,0 +1,28 @@
+class Categories < Sequel::Migration
+
+  def up
+    create_table :categories do
+      primary_key :id
+      Integer :parent_id, :references => :categories
+      String :name
+      String :key
+
+      index [:parent_id, :id]
+    end
+
+    alter_table :texts do
+      add_column :category_id, Integer, :references => :categories
+
+      add_index [:category_id, :id]
+    end
+
+  end
+
+  def down
+    alter_table :texts do
+      drop_column :category_id
+    end
+    drop_table :categories
+  end
+
+end

data/lib/lumix/schema/003_add_fulltagged.rb

@@ -0,0 +1,15 @@
+class AddFulltagged < Sequel::Migration
+
+  def up
+    alter_table :texts do
+      add_column :fulltagged, String
+    end
+  end
+
+  def down
+    alter_table :texts do
+      drop_column :fulltagged
+    end
+  end
+
+end

data/lib/lumix/schema/004_create_lookup_tables.rb

@@ -0,0 +1,44 @@
+class CreateLookupTables < Sequel::Migration
+
+  def up
+    create_table :tags do
+      primary_key :id
+      String :tag
+
+      index :tag, :unique => true
+    end
+
+    create_table :words do
+      primary_key :id
+      String :word
+
+      index :word, :unique => true
+    end
+
+    create_table :tokens do
+      primary_key :id
+      Integer :text_id, :references => :texts
+
+      Integer :position
+      Integer :tag_id, :references => :tags
+      Integer :word_id, :references => :words
+
+      Integer :src_begin
+      Integer :src_end
+      Integer :tagged_begin
+      Integer :tagged_end
+
+      index [:text_id, :position], :unique => true
+      index :word_id
+      index :tag_id
+    end
+
+  end
+
+  def down
+    drop_table :tokens
+    drop_table :words
+    drop_table :tags
+  end
+
+end

data/lib/lumix/slow_search.rb

@@ -0,0 +1,104 @@
+module Lumix
+
+  class SlowSearch
+    TAGGED = /([^\s\|]+)\|(\S+)/m        # Xxx|YYY
+
+    def initialize(db, progress)
+      @db = db
+      @progress = progress
+    end
+
+    def concurrent_link?
+      true
+    end
+
+    def link_text(id)
+      t = TaggedText[id]
+      text = t.text
+      puts "Linking text #{t.filename}"
+
+      src_last = 0
+      position = 0
+      assoc = []
+      t.tagged.scan(TAGGED) do |word, tag|
+        tagged_begin = $~.begin(0)
+        tagged_end = $~.end(0)
+
+        word_re = Regexp.new(Regexp.escape(word).gsub(/_/, '\s*'))
+        src_match = text[src_last..-1].match(word_re) # find the word
+        if src_match
+          src_begin = src_last + src_match.begin(0)
+          src_end = src_last + src_match.end(0)
+
+          src_last = src_end
+          assoc << {:text_id => id, :position => position, :src_begin => src_begin, :src_end => src_end, :tagged_begin => tagged_begin, :tagged_end => tagged_end}
+        else
+          STDERR.puts "Could not find match for '#{word}' in text #{t.filename}"
+          `echo '#{t.filename}:#{tagged_begin}:#{src_last} unmatched "#{word}"' >> unlinked.lst`
+          return nil
+        end
+        position += 1
+      end
+      @db[:assoc].multi_insert(assoc)
+    rescue => e
+      STDERR.puts e
+      STDERR.puts e.backtrace
+      raise e
+    end
+
+    def create_filter
+      @filter ||= Filter.new('')
+    end
+
+    def find(filter, &block)
+      yield_text = block && block.arity >= 1
+      yield_tagged = block && block.arity >= 2
+
+      prog = Progress.new(:search, TaggedText.count, filter)
+      @progress[prog]
+
+      re = Filter.to_re(filter)
+
+      index = 0
+      TaggedText.inject(0) do |result, t|
+        fname = File.basename(t.filename)
+
+        # matches to ranges
+        results = []
+        t.tagged.scan(re) do |hit|
+          t_begin = $~.begin(0)
+          t_end = $~.end(0)
+          # TODO decouple database operations for performance
+          results << find_range(t.id, t_begin, t_end, yield_text)
+        end
+
+        result += results.inject(0) do |res, f|
+          if yield_tagged
+            tagged_snippet = TextSnippet.new(fname, t.tagged, f[:tagged_begin].to_i, f[:tagged_end].to_i)
+            if yield_text
+              text_snippet = TextSnippet.new(fname, t.text, f[:src_begin].to_i, f[:src_end].to_i)
+              yield text_snippet, tagged_snippet
+            else
+              yield tagged_snippet
+            end
+          end
+          res += 1
+        end
+        @progress[prog, (index += 1)]
+        result
+      end
+    end
+
+    def find_range(t_id, t_begin, t_end, process_original)
+      if process_original
+        ds = @db[:assoc].filter(:text_id => t_id).filter{tagged_end >= t_begin}.filter{tagged_begin < t_end}
+        ds.select{[{min(:src_begin) => :src_begin},{ max(:src_end) => :src_end}, {min(:tagged_begin) => :tagged_begin}, {max(:tagged_end) => :tagged_end}]}.first
+      else
+        {:tagged_begin => t_begin, :tagged_end => t_end}
+      end
+    end
+
+  end
+
+  SearchStrategy = SlowSearch
+end

data/lib/lumix/text_snippet.rb

@@ -0,0 +1,29 @@
+module Lumix
+  
+  class TextSnippet
+    attr_reader :name, :text, :begin, :end
+    def initialize(name, text, first, last)
+      @name = name
+      @text = text
+      @begin = first
+      @end = last
+    end
+    def to_s
+      cleanup(@text[@begin...@end])
+    end
+    def left(context = 5)
+      ctx = [@begin - context * 10, 0].max
+      @text[ctx...@begin].split(/\s+/).last(context).join(' ')# =~ /((\S+\s+){0,#{context}}\S*)\z/m
+      #cleanup($1)
+    end
+    def right(context = 5)
+      ctx = [@end + context * 10, @text.size].min
+      @text[@end..ctx].split(/\s+/).first(context).join(' ')# =~ /\A(\S*(\s+\S+){0,#{context}})/m
+      #cleanup($1)
+    end
+    def cleanup(txt)
+      txt.gsub(/\s+/, ' ')
+    end
+  end
+
+end

data/lib/lumix/textprocessing.rb

@@ -0,0 +1,108 @@
+$KCODE='UTF-8'
+
+require 'cgi'
+require 'soap/wsdlDriver'
+#require 'curb'
+#require 'savon'
+require 'lumix/charset' unless RUBY_ENGINE =~ /maglev/i
+
+class TextProcessing
+
+  attr_accessor :lang
+
+  def initialize(lang = 'ro')
+    @lang = lang
+  end
+
+  def rpc
+#    Thread.current[:rpc] ||= begin
+#      wsdl = SOAP::WSDLDriverFactory.new('http://www.racai.ro/webservices/TextProcessing.asmx?WSDL')
+#      wsdl.create_rpc_driver
+#      Savon::Client.new('http://www.racai.ro/webservices/TextProcessing.asmx?WSDL')
+#    end
+    @rpc ||= SOAP::WSDLDriverFactory.new('http://www.racai.ro/webservices/TextProcessing.asmx?WSDL').create_rpc_driver
+  end
+
+  # the core processing routing using the webservice
+  def process(text)
+    response = rpc.Process(:input => text.to_utf, :lang => lang)
+    response.processResult
+#    response = rpc.request(:process) do
+#      soap.body = {:input => text, :lang => lang}
+#    end
+#    response.to_hash[:process_response][:process_result]
+  end
+
+  def cleanup(file)
+    @entities ||= HTMLEntities.new
+    @entities.decode()
+  end
+
+  # inserts "tagged" as the second to last part in the filename and as parent folder
+  # e.g.
+  #   test.txt -> tagged/test.tagged.txt
+  # special case when no extension is present:
+  #   README -> README.tagged
+  def create_tagged_filename(infile)
+    path = infile.split(/\//)
+
+    # take care of the filename...
+    components = path.pop.split(/\./)
+    position = [1, components.size-1].max
+    components.insert position, 'tagged'
+    path.push components.join('.')
+
+    # ...and of the path
+    path.insert -2, 'tagged'
+    path.join '/'
+  end
+
+  def to_filelist(*files)
+    files = files.flatten.map do |filename|
+      if File.directory?  filename
+        Dir.glob File.join(filename, '**/*') # add all files from that directory
+      else
+        filename
+      end
+    end.flatten.compact.uniq # make sure every file is only processed once
+    files.delete_if { |filename| File.directory?(filename) ||  filename['.tagged']} # remove remaining folders
+  end
+
+  def process_stdin
+    puts process($stdin.read)
+  end
+
+  # takes the text from infile and outputs the result into the outfile
+  def process_file(infile, outfile = create_tagged_filename(infile))
+    result = process(File.read(file).to_utf)
+    File.open(outfile, 'w') do |out|
+      out.write result
+    end
+  end
+
+end
+
+
+# process the args if called as main script
+if __FILE__ == $0
+  args = ARGV
+  tp = if args.first == '-lang'
+    args.shift
+    TextProcessing.new(args.shift)
+  else
+    TextProcessing.new
+  end
+
+  if args.empty?
+    tp.process_stdin
+  else
+    files = tp.to_filelist(args)
+
+    puts "Processing files:"
+    for infile in files
+      outfile = tp.create_tagged_filename(infile)
+      puts "#{infile} -> #{outfile}"
+      tp.process_file(infile, outfile) unless File.exist?(outfile)
+    end
+  end
+end