weft-qda 0.9.6 → 0.9.8

data/lib/weft/filters/indexers.rb

@@ -0,0 +1,137 @@
+module QDA
+  class Indexer
+    attr_reader :cursor
+    def initialize()
+      @cursor = 0
+    end
+
+    def index(str)
+      prepare(str)
+      str.each_line { | line | feed(line) }
+    end
+
+    def terminate()
+    end
+
+    def prepare(content)
+    end
+
+    def feed(line)
+      @cursor += line.length
+    end
+  end
+  # An indexer which records the position of words for later reverse
+  # retrieval
+  class WordIndexer <  Indexer
+    attr_reader :words
+    # includes accented latin-1 characters
+    WORD_TOKENIZER = /[\w\xC0-\xD6\xD8-\xF6\xF8-\xFF][\w\xC0-\xD6\xD8-\xF6\xF8-\xFF\']+/
+    def initialize()
+      super
+      @words = Hash.new { | h, k | h[k] = [] }
+    end
+
+    def feed(line)
+      line.scan( WORD_TOKENIZER ) do | word |
+        next if word.length == 1
+        if word.respond_to?(:offset)
+          offset = cursor + word.offset
+        else
+          offset = cursor + Regexp.last_match.begin(0)
+        end
+        @words[word.to_s].push(offset)
+      end
+      super
+    end
+  end
+
+  # An indexer that uses text patterns to identify, for example,
+  # passages by a particular speaker, or text headings. 
+  # The indexer can recognise a number of different types of codes,
+  # each denoted by a pattern of punctuation in a line of text. A
+  # default coder recognises the following
+  # A 'Heading', marked by a line **NAME OF HEADING**
+  # A 'Speaker', marked by a line SpeakerName:
+  # 
+  # After the filter has run, the results of the coding can be
+  # retrieved by calling Autocoder#codes
+  # This is a hash of codetype names to inner hashes of codevalue names
+  # (strings) to QDA::Codesets corresponding to them.
+  class AutoCoder < Indexer
+    STANDARD_TRIGGER_RULES = {
+      /^(\w+)\:\s*$/   => 'Speaker',
+      /^\*\*(.*)\*\*$/ => 'Heading'
+    }
+
+    attr_reader :codes
+    # +rules+ should be a hash of string keys, naming types of autocode
+    # (e.g. "Speaker", "Heading", "Topic") mapped to values, which
+    # should be regular expressions specifying how the start of such a
+    # code should be recognised.
+    # For example, to find topics marked by the characters '##' at the
+    # start of the line:
+    # 'Heading' => /^##(.*)$/
+    def initialize(rules = STANDARD_TRIGGER_RULES)
+      super()
+      @trigger_rules = rules
+      @codes      = Hash.new { | h, k | h[k] = {} }
+      @curr_codes = {}
+    end
+
+    # check a line of document content for triggers
+    def feed(line)
+      @trigger_rules.each do | rule, type |
+        if match = rule.match(line)
+          trigger(cursor, type, match[1]) 
+        end
+      end
+      super
+    end
+
+    # take action on finding a autocode marker
+    def trigger(cursor, group, codename)
+      # save any previous code that was being done for this group
+      store(group) if @curr_codes[group]
+      new_codeset = get_code(group, codename)
+      @curr_codes[group] = [ new_codeset, cursor ]
+    end
+    private :trigger
+
+    # returns the code name +codename+ within the group +group+,
+    # creating a new empty category
+    def get_code(group, codename)
+      return @codes[group][codename] if @codes[group][codename]
+      @codes[group][codename] = QDA::CodeSet.new()
+    end
+
+    # Returns the names and codesets for autocodes in group +group+
+    # in a series of pairs
+    def each_autocode(group)
+      @codes[group].each { | name, codeset | yield name, codeset }
+    end
+
+    # alters all the stored coding in this autocoder so that it refers
+    # to the document identified by +docid+
+    def apply(docid)
+      @codes.values.each do | group  |
+        group.values.each do | codeset | 
+          codeset.map! { | x | x.docid = docid; x }
+        end
+      end
+    end
+
+    # finish up all currently active coding in this autocoder
+    def terminate()
+      @curr_codes.each_key { | group | store(group) }
+    end
+
+    # finish the coding for the current code being used among +group+
+    def store(group)
+      codeset, start = @curr_codes[group]
+      # -1 here is a placeholder 
+      terminus = cursor - start
+      codeset.add( Code.new(-1, start, terminus) )
+    end
+    private :store
+  end
+end

data/lib/weft/filters/input.rb

@@ -0,0 +1,118 @@
+module QDA
+module Filters
+  class DocumentTextFilter
+    IMPORT_CLASS = Document
+    MEDIA_TYPE = 'text/plain'
+    
+    def run(content_or_file)
+      content = respond_to?(:read) ? read(content_or_file) :
+                                     content_or_file
+  	  doc = QDA::Document.new('', '')
+      # signal to indexers we're about to start
+      @indexers.each { | indexer | indexer.prepare(content) }
+	  content.each_line do | line |
+        doc.append(line.to_s.chomp)
+        @indexers.each { | indexer | indexer.feed(line) }
+	  end
+      @indexers.each { | indexer | indexer.terminate() }
+	  doc.create
+	  doc
+    end
+    
+    
+    attr_reader :cursor
+    
+	def initialize()
+      @cursor = 0
+      @indexers = []
+	end
+	
+    def add_indexer(indexer)
+      unless indexer.respond_to?(:feed)
+        raise "Document indexers should have a feed method"
+      end
+      @indexers.push(indexer)
+    end
+  end
+  
+  module TextReader
+	def read(filename)
+      File.read(filename)
+	end
+  end
+  
+  class TextFileDocumentFilter < DocumentTextFilter
+    EXTENSIONS  = ['txt']
+    DESCRIPTION = 'Plain Text Files'
+    include TextReader
+    Filters.register_filter(self)
+  end
+  
+  module PDFReader
+    # This tests if we are on windows - will raise an exception if used
+    # on *nix platforms
+    begin
+      require 'weft/filters/win32backtick.rb'
+    rescue LoadError
+    end
+  
+  
+    maybe_pdf = 'pdftotext'
+    begin
+      if defined? Win32Process
+        # the -v option for pdftotext writes to STDERR
+        ignore, test = Win32Process.backtick("#{maybe_pdf} -v")
+      else
+        test = `#{maybe_pdf} -v 2>&1`
+      end
+
+      if test =~ /pdftotext version 3/
+        PDF_TO_TEXT_EXEC = maybe_pdf
+      else
+        warn 'PDFtotext Version 3 not found in path; ' <<
+             'PDF Filters will not be available'
+      end
+    end
+
+    def read(file)
+      file = "\"#{file}\"" if file =~ /\s/
+      cmd = "#{PDF_TO_TEXT_EXEC} -nopgbrk #{file} -"
+      if defined? Win32Process
+        do_read_win32(cmd)
+      else
+        do_read_unixy(cmd)
+      end
+    end
+    
+    private
+    def do_read_win32(cmd)
+      out, err = Win32Process.backtick(cmd)
+      return out unless out.empty?
+      if err =~ /Copying of text from this document is not allowed/
+        raise IOError.new("Could not extract PDF text: this PDF is locked")
+      else
+        raise IOError.new("Could not extract PDF text: #{err}")
+      end      
+    end
+    
+    def do_read_unixy(cmd)
+      content = `#{cmd} 2>&1`        
+      case $CHILD_STATUS
+      when 0
+        return content
+      when 3
+        raise IOError.new("Could not extract PDF text: this PDF is locked")
+      else
+        raise IOError.new("Could not extract PDF text: #{content}")
+      end
+    end
+  end
+  
+  class PDFFileDocumentFilter < DocumentTextFilter
+    EXTENSIONS = ['pdf']
+    DESCRIPTION = 'PDF Files'
+    include PDFReader
+    Filters.register_filter(self) if defined?(PDFReader::PDF_TO_TEXT_EXEC)
+  end
+end
+end

data/lib/weft/filters/output.rb

@@ -0,0 +1,101 @@
+module QDA
+  module Filters
+
+    begin
+      require 'csv'
+    rescue LoadError
+    end
+
+    begin
+      require 'PageTemplate/parser'
+      require 'PageTemplate/commands'
+      require 'weft/filters/templates.rb'
+      # These features won't be used if not available
+    rescue LoadError
+    end
+
+    class OutputFilter
+      def initialize(app = nil)
+        @app = app
+      end
+    end
+
+    # Inherit, don't instantiate this class directly
+    class DocumentOutputFilter < OutputFilter
+      EXPORT_CLASS = Document
+      def DocumentOutputFilter.def_name(doc) 
+        'document-' << doc.title.gsub(/\s+/, '_').gsub(/\W/,'')
+      end
+    end
+
+    # A filter that formats a Category for output, including supplying the
+    # text marked by the category
+    class CategoryOutputFilter < OutputFilter
+      EXPORT_CLASS = Category
+      def CategoryOutputFilter.def_name(cat) 
+        'category-' << cat.name.gsub(/\s+/, '_').gsub(/\W/,'')
+      end
+      
+      def variables(obj)
+        { 'text' => @app.get_text_at_category(obj) }
+      end
+    end
+    
+    class CodeReviewOutputFilter < OutputFilter
+      EXPORT_CLASS = CodeReview
+      def self.def_name(cr)
+        'codereview-' << cr.dbid.to_s
+      end
+    end
+    
+    class CodeReviewCSVOutput < CodeReviewOutputFilter
+      EXTENSION  = 'csv'
+      DESCRIPTION = 'Code Review as spreadsheet'
+      def write(cr, file = STDOUT, mode = nil )
+        mode ||= cr.count_method
+        CSV::Writer.generate(file) do | csv |
+          cr.output_rows.each { | r | csv << r }
+        end
+      end
+      Filters.register_filter(self) if defined?(CSV)
+    end
+    
+    tfiles =  File.join(TemplatedOutput::TEMPLATES_DIR, '*.*') 
+
+    # autoload template-based output filters from the shared templates dir
+    # this is located in /path/to/weft/share/templates.
+    # Each of these special template files should contain a number of comment
+    # header lines, followed by a text template.
+    # Specifications for the template are given as comment lines as follows:
+    #  DESCRIPTION [required]  : short description of template's use, for UI
+    #  EXPORT_CLASS [required] : QDA class templated by this file
+    #  EXTENSION [optional ]   : what file extension exported; default, 
+    #                             same extension as the template file
+    #  CLASSNAME [ optional ]  : make a reference to this class in the module
+    #                            QDA::Filters
+    Dir.glob( tfiles ).each do | tfile |
+      tpl_class = nil
+      File.foreach(tfile) do | line |
+        case line
+        when /^#\s*EXPORT_CLASS\s*:(.*)$/
+          base_class = Filters.const_get( $1.strip << 'OutputFilter' )
+          tpl_class  = Class.new( base_class )
+        when /^#\s*(DESCRIPTION|EXTENSION)\s*:(.*)$/
+          tpl_class.const_set($1.strip, $2)
+        when /^#\s*CLASSNAME\s*:(.*)$/
+          Filters.const_set($1.strip, tpl_class)
+        when /^#/
+            # ignore
+        else
+          break
+        end
+      end
+      next unless tpl_class
+      tpl_class.const_set( 'TEMPLATE_FILE', File.basename(tfile) )
+      if not defined? tpl_class.EXTENSION
+        tpl_class.const_set( 'EXTENSION', File.extname(tfile) )
+      end
+      tpl_class.instance_eval { include TemplatedOutput }
+    end
+  end  
+end

data/lib/weft/filters/templates.rb

@@ -0,0 +1,80 @@
+module QDA::Filters
+  module Templates
+    class CommentStrippingFileSource < PageTemplate::FileSource
+      # return the template content of the file named +name+. Uses the standard
+      # PageTemplate FileSource mechanism for searching directories etc, but 
+      # removes any comment lines (all lines at the beginning of the document
+      # starting with a comment marker '#') before returning the content.
+      def get(name)
+        # all stuff starting from the beginning up to the first newline that is
+        # not followed by a comment character
+        super(name).sub(/\A.*?\n(?!#)/m, '')
+      end
+    end
+    
+    class ExtendedPreprocessor < PageTemplate::DefaultPreprocessor
+    class << self
+      # shorter and more ruby-ish names for the PageTemplate methods
+      alias :html :escapeHTML
+      alias :uri :escapeURI
+
+      # Takes string +string+ and splits it into HTML paragraphs, taking a blank
+      # line and a newline as marking the start of a paragraph. 
+      # Does HTML escaping of the paragraph contents
+      def html_paras(string)
+        string.split(/\n\s*\n/).inject('') do | out, para |
+          out << "<p>" << html(para) << "</p>\n"
+        end
+      end
+    end
+  end
+
+  end
+  
+  
+  # Included in output classes to enable them to be output via template (using
+  # PageTemplate under the hood). 
+  module TemplatedOutput
+      include Templates
+      TEMPLATES_DIR = File.join( WEFT_SHAREDIR, 'templates' )
+      # write templated output of the QDA object +obj+ (eg a Category, a 
+      # Document or a CodeReview) to the file or io +file+.
+      def write(obj, file = STDOUT)
+        tpl = PageTemplate::Parser.new('source' => CommentStrippingFileSource,
+                                       'include_paths' => TEMPLATES_DIR,
+                                       'preprocessor' => ExtendedPreprocessor )
+        tpl.load(self.class::TEMPLATE_FILE)
+        variables(obj).each { | k, v | tpl[k] = v }
+        file.puts( tpl.output() )
+      end
+      
+      # Returns a hash containing the named variables that should be templated.
+      # By default simply returns a hash containing the named variable 'obj'
+      # which contains the object being templated.
+      # If a variables() method is defined in the superclass (eg CategoryOutput)
+      # then the results from that are merged with the default. For example, 
+      # CategoryOutput includes the named variable "text" which contains all
+      # the text marked by the category.
+      def variables(obj)
+        super.merge( 'obj' => obj )
+      rescue NoMethodError
+        { 'obj' => obj  }
+      end
+      
+      
+      # Whenever this module is included in another class, it automatically
+      # checks whether a valid filter has been defined. If one has (by defining
+      # what QDA class it can output, what template file it should use, and
+      # what file extension it creates), then the filter is registered for use
+      # with the app (so it can be queried through +Filters.export_filters()+
+      def TemplatedOutput.included(other)
+        if other.is_a?(Class) and
+          defined?(other::EXPORT_CLASS) and
+          defined?(other::TEMPLATE_FILE) and
+          defined?(other::EXTENSION) and
+          defined?(PageTemplate)
+          QDA::Filters.register_filter(other)
+        end
+      end
+    end
+end

data/lib/weft/filters/win32backtick.rb

@@ -0,0 +1,246 @@
+require 'Win32API'
+
+module QDA
+  module Filters
+    # Used only on windows to enable calling other executables without the
+    # annoying command-prompt box that pops up when using Ruby backticks in
+    # a script running under rubyw.
+    # 
+    # Note - most of this code written by S Kroeger, see
+    # http://blade.nagaokaut.ac.jp/cgi-bin/scat.rb/ruby/ruby-talk/155684
+    module Win32Process
+      NORMAL_PRIORITY_CLASS = 0x00000020
+      STARTUP_INFO_SIZE = 68
+      PROCESS_INFO_SIZE = 16
+      SECURITY_ATTRIBUTES_SIZE = 12
+
+      ERROR_SUCCESS = 0x00
+      FORMAT_MESSAGE_FROM_SYSTEM = 0x1000
+      FORMAT_MESSAGE_ARGUMENT_ARRAY = 0x2000
+
+      HANDLE_FLAG_INHERIT = 1
+      HANDLE_FLAG_PROTECT_FROM_CLOSE =2
+
+      STARTF_USESHOWWINDOW = 0x00000001
+      STARTF_USESTDHANDLES = 0x00000100
+      
+      class << self
+        def raise_last_win_32_error
+          errorCode = Win32API.new("kernel32", "GetLastError", [], 'L').call
+          if errorCode != ERROR_SUCCESS
+            params = [
+              'L', # IN DWORD dwFlags,
+              'P', # IN LPCVOID lpSource,
+              'L', # IN DWORD dwMessageId,
+              'L', # IN DWORD dwLanguageId,
+              'P', # OUT LPSTR lpBuffer,
+              'L', # IN DWORD nSize,
+              'P', # IN va_list *Arguments
+            ]
+
+            formatMessage = Win32API.new("kernel32", "FormatMessage", params, 'L')
+            msg = ' ' * 255
+            msgLength = formatMessage.call(FORMAT_MESSAGE_FROM_SYSTEM +
+                                              FORMAT_MESSAGE_ARGUMENT_ARRAY, '', errorCode, 0, msg, 255, '')
+
+            msg.gsub!(/\000/, '')
+            msg.strip!
+            raise msg
+          else
+            raise 'GetLastError returned ERROR_SUCCESS'
+          end
+        end
+
+        def create_pipe # returns read and write handle
+          params = [
+            'P', # pointer to read handle
+            'P', # pointer to write handle
+            'P', # pointer to security attributes
+            'L'] # pipe size
+
+          createPipe = Win32API.new("kernel32", "CreatePipe", params, 'I')
+
+          read_handle, write_handle = [0].pack('I'), [0].pack('I')
+          sec_attrs = [SECURITY_ATTRIBUTES_SIZE, 0, 1].pack('III')
+
+          raise_last_win_32_error if createPipe.Call(read_handle,
+                                                      write_handle, sec_attrs, 0).zero?
+
+          [read_handle.unpack('I')[0], write_handle.unpack('I')[0]]
+        end
+
+        def set_handle_information(handle, flags, value)
+          params = [
+            'L', # handle to an object
+            'L', # specifies flags to change
+            'L'] # specifies new values for flags
+
+          setHandleInformation = Win32API.new("kernel32",
+                                               "SetHandleInformation", params, 'I')
+          raise_last_win_32_error if setHandleInformation.Call(handle,
+                                                                flags, value).zero?
+          nil
+        end
+
+        def close_handle(handle)
+          closeHandle = Win32API.new("kernel32", "CloseHandle", ['L'], 'I')
+          raise_last_win_32_error if closeHandle.call(handle).zero?
+        end
+
+        def create_process(command, stdin, stdout, stderror)
+          params = [
+            'L', # IN LPCSTR lpApplicationName
+            'P', # IN LPSTR lpCommandLine
+            'L', # IN LPSECURITY_ATTRIBUTES lpProcessAttributes
+            'L', # IN LPSECURITY_ATTRIBUTES lpThreadAttributes
+            'L', # IN BOOL bInheritHandles
+            'L', # IN DWORD dwCreationFlags
+            'L', # IN LPVOID lpEnvironment
+            'L', # IN LPCSTR lpCurrentDirectory
+            'P', # IN LPSTARTUPINFOA lpStartupInfo
+            'P']  # OUT LPPROCESS_INFORMATION lpProcessInformation
+
+          startupInfo = [STARTUP_INFO_SIZE, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+            STARTF_USESTDHANDLES | STARTF_USESHOWWINDOW, 0,
+            0, 0, stdin, stdout, stderror].pack('IIIIIIIIIIIISSIIII')
+
+          processInfo = [0, 0, 0, 0].pack('IIII')
+          command << 0
+
+          createProcess = Win32API.new("kernel32", "CreateProcess", params, 'I')
+          cp_args = [ 0, command, 0, 0, 1, 0, 0, 0, startupInfo, processInfo ]
+          raise_last_win_32_error if createProcess.call(*cp_args).zero?
+
+          hProcess, hThread, 
+          dwProcessId, dwThreadId = processInfo.unpack('LLLL')
+
+          close_handle(hProcess)
+          close_handle(hThread)
+
+          [dwProcessId, dwThreadId]
+        end
+
+        def write_file(hFile, buffer)
+          params = [
+            'L', # handle to file to write to
+            'P', # pointer to data to write to file
+            'L', # number of bytes to write
+            'P', # pointer to number of bytes written
+            'L'] # pointer to structure for overlapped I/O
+
+          written = [0].pack('I')
+          writeFile = Win32API.new("kernel32", "WriteFile", params, 'I')
+
+          raise_last_win_32_error if writeFile.call(hFile, buffer, buffer.size,
+                                                     written, 0).zero?
+
+          written.unpack('I')[0]
+        end
+
+        def read_file(hFile)
+          params = [
+            'L', # handle of file to read
+            'P', # pointer to buffer that receives data
+            'L', # number of bytes to read
+            'P', # pointer to number of bytes read
+            'L'] #pointer to structure for data
+
+          number = [0].pack('I')
+          buffer = ' ' * 255
+
+          readFile = Win32API.new("kernel32", "ReadFile", params, 'I')
+          return '' if readFile.call(hFile, buffer, 255, number, 0).zero?
+
+          buffer[0...number.unpack('I')[0]]
+        end
+
+        def peek_named_pipe(hFile)
+          params = [
+            'L', # handle to pipe to copy from
+            'L', # pointer to data buffer
+            'L', # size, in bytes, of data buffer
+            'L', # pointer to number of bytes read
+            'P', # pointer to total number of bytes available
+            'L'] # pointer to unread bytes in this message
+
+          available = [0].pack('I')
+          peekNamedPipe = Win32API.new("kernel32", "PeekNamedPipe", params, 'I')
+
+          return -1 if peekNamedPipe.Call(hFile, 0, 0, 0, available, 0).zero?
+
+          available.unpack('I')[0]
+        end
+      end
+
+      class Win32popenIO
+        def initialize (hRead, hWrite, hError)
+          @hRead  = hRead
+          @hWrite = hWrite
+          @hError = hError
+        end
+
+        def write data
+          Win32Process::write_file(@hWrite, data.to_s)
+        end
+
+        def read
+          sleep(0.01) while Win32Process::peek_named_pipe(@hRead).zero?
+          Win32Process::read_file(@hRead)
+        end
+
+        def read_all
+          all = ''
+          until (buffer = read).empty?
+            all << buffer
+          end
+          all
+        end
+
+        def read_err
+          sleep(0.01) while Win32Process::peek_named_pipe(@hError).zero?
+          Win32Process::read_file(@hError)
+        end
+
+        def read_all_err
+          all = ''
+          until (buffer = read_err).empty?
+            all << buffer
+          end
+          all
+        end
+      end
+
+      # The only useful public method in this class - receives a command line,
+      # and returns the output content and error content as a pair of strings.
+      # No shell expansion is carried out on the command line string.
+      # 
+      #  output, errors = Win32Process::backtick('ls')
+      def self.backtick(command)
+        # create 3 pipes
+        child_in_r, child_in_w = create_pipe
+        child_out_r, child_out_w = create_pipe
+        child_error_r, child_error_w = create_pipe
+
+        # Ensure the write handle to the pipe for STDIN is not inherited.
+        set_handle_information(child_in_w, HANDLE_FLAG_INHERIT, 0)
+        set_handle_information(child_out_r, HANDLE_FLAG_INHERIT, 0)
+        set_handle_information(child_error_r, HANDLE_FLAG_INHERIT, 0)
+
+        processId, threadId = create_process( command, 
+                                              child_in_r, 
+                                              child_out_w, 
+                                              child_error_w )
+        # we have to close the handles, so the pipes terminate with the process
+        close_handle(child_in_r)
+        close_handle(child_out_w)
+        close_handle(child_error_w)
+        close_handle(child_in_w)
+        io = Win32popenIO.new(child_out_r, child_in_w, child_error_r)
+ 
+        out = io.read_all().gsub(/\r/, '')
+        err = io.read_all_err().gsub(/\r/, '')
+        return out, err
+      end
+    end
+  end
+end