weft-qda 0.9.6

data/lib/weft.rb

@@ -0,0 +1,21 @@
+require 'weft/filters'
+require 'weft/document'
+require 'weft/category'
+require 'weft/backend'
+require 'weft/application'
+
+begin
+  require 'weft/WEFT-VERSION-STRING'
+  WEFT_VERSION = QDA::Version.new(WEFT_VERSION_STRING)
+  # rs2exed but from local CVS branch
+  if WEFT_VERSION == QDA::Version.default_version()
+    WEFT_TESTING = true
+  else
+    # release or testing version from CVS checkout 
+    WEFT_TESTING = false
+  end
+rescue LoadError
+  WEFT_VERSION = QDA::Version.default_version()
+  WEFT_VERSION_STRING = '[unreleased version]'
+  WEFT_TESTING = true
+end

data/lib/weft/WEFT-VERSION-STRING.rb

@@ -0,0 +1 @@
+WEFT_VERSION_STRING = '0.9.6'

data/lib/weft/application.rb

@@ -0,0 +1,130 @@
+require 'observer'
+
+module QDA
+  class Version
+    attr_reader :major, :minor, :release
+    def Version.default_version()
+      self.new('0.0.0')
+    end
+    
+    def initialize(str)
+      if str =~ /(\d+)\.(\d+)\.(\d+)\s*/
+        @major, @minor, @release = $1, $2, $3
+      else
+        raise ArgumentError.new("Invalid version string #{str}")
+      end
+    end
+
+    def to_s
+      [@major, @minor, @release].join('.')
+    end
+
+    def ==(other)
+      if other.kind_of?(String)
+        other = Version.new(other)
+      end
+      self.major == other.major && 
+        self.minor == other.minor && 
+        self.release == other.release
+    end
+  end
+
+  class Application
+    include Observable
+    def initialize(observer = nil)
+      add_observer(observer) if observer
+      @dirty = false
+    end
+
+    # creates a completely empty new application / project, using the
+    # backend +backend+, to be intialized with args +args+
+	def Application::new_virgin(backend, args, observer = nil)
+	  app = new()
+	  app.extend( backend )
+	  app.start(args)
+	  app.install_clean()
+	  app
+	end
+    
+    # create some basic nodes
+    def set_up()
+      save_category( Category.new('CATEGORIES', nil) )
+      save_category( Category.new('SEARCHES', nil) )
+      save_preference( 'CreateVersion', WEFT_VERSION )
+      save_preference( 'CreateVersionString', WEFT_VERSION_STRING )
+      undirty!
+    end
+
+	def each_doc()
+	  get_all_docs().each { | doc | yield doc }
+	end
+	
+	# are we in a state where we need saving - should be overridden in subclass
+	def dirty?()
+      @dirty
+	end
+
+    def dirty!()
+      changed if not dirty?
+      @dirty = true
+      notify_observers(@dirty)
+    end
+
+    def undirty!()
+      changed if dirty?
+      @dirty = false
+      notify_observers(@dirty)
+    end
+
+	# is it up and running
+	def started?
+	  true
+	end
+
+	# signal to clear up - should this be the level at which an exception
+	# is raised - no, should probably be at the gui level
+	def finish(force = false)
+	  if ! consistent? and ! force
+		raise "Not ready to be saved"
+	  end
+	end
+
+    def query_segment(function, *args)
+      case function
+      when "IS CODED BY"
+        category = get_category(args[0])
+        return get_text_at_category( category )
+      when "CONTAINS WORD"
+        return get_search_fragments( args[0], :wrap_both => 100 )
+      else
+        raise RuntimeError.new("Unknown function '#{function}' in query")
+      end
+    end
+    private :query_segment
+
+    # executes a query, which is a series of descriptions of text
+    # functions (eg 'CODED BY "category x"') and operators describing
+    # how to combine them('AND', 'NOT', 'Or')
+    def do_query(*query)
+      text_1 = query_segment(query.shift, query.shift)
+
+      # work rightwards through the query, doing various kinds of
+      # combination of the result sets retrieved
+      while op = query.shift
+        return text_1 if op.empty?
+        text_2 = query_segment(query.shift, query.shift)
+        if op == 'AND'
+          text_1.join(text_2)
+        elsif op == 'OR'
+          text_1.merge(text_2)
+        elsif op =~ /(AND )?NOT/
+          text_1.remove(text_2)
+        else
+          raise RuntimeError.new("Unknown operator '#{op}' in query")
+        end
+      end
+
+      return text_1
+    end
+  end
+end

data/lib/weft/backend.rb

@@ -0,0 +1,39 @@
+module QDA
+  module Backend
+	autoload :MySQL, 'weft/backend/mysql'
+	# autoload :SQLite, 'backend/sqlite'
+	require 'weft/backend/sqlite'
+	# autoload :N6, 'backend/n6'
+	require 'weft/backend/n6'
+	autoload :RubyNative, 'weft/backend/marshal'
+
+	module Abstract
+	  # receive arguments and make any connection required to the
+	  # storage source
+	  def start(args = {})
+		# raise "virtual"
+	  end
+	  
+	  # load a specific document
+	  def get_doc(doctitle)
+		raise "virtual"	
+	  end
+
+	  # an array of all the documents - should this include TEXT
+	  def get_all_docs()
+		[]
+	  end
+
+	  # all categories in a tree structure, the root nodes are returned
+	  def get_all_categories()
+		[ Category.new('') ]
+	  end
+
+	  # save changes
+	  def save()
+		raise "virtual"	
+	  end
+
+	end
+  end
+end

data/lib/weft/backend/marshal.rb

@@ -0,0 +1,26 @@
+# storage backend using Ruby's Marshal module
+module QDA
+  module Backend
+	module MySQL
+	  def connect(args)
+		@basedir = args[:basedir]
+		File.open(@basedir + 'docs') { | f | @docs = Marshal.load(f) }
+		@docs.values.each { | doc | doc.source = self }
+
+		File.open(@basedir + 'nodes') { | f | @nodes = Marshal.load(f) }
+		@nodes.each { | node | node.source = self }
+		@root_node = @nodes.detect { | node | node.is_a?(Nudist::RootNode) }
+	  end
+
+	  def save()
+		@docs.values.each { | doc | doc.source = nil }
+		File.open(@basedir + 'docs', 'w') { | f | Marshal.dump(@docs, f) }
+		@docs.values.each { | doc | doc.source = self }
+
+		@nodes.each { | node | node.source = nil }
+		File.open(@basedir + 'nodes', 'w') { | f | Marshal.dump(@nodes, f) }
+		@nodes.each { | node | node.source = self }
+	  end
+	end
+  end
+end

data/lib/weft/backend/mysql.rb

@@ -0,0 +1,267 @@
+require 'dbi'
+
+# storage backend using MYSQL-dbi module
+module QDA
+  module Backend
+	module MySQL
+	  # load up the database connection
+	  def connect(dbh)
+		@dbh = dbh
+	  end
+
+	  # 
+	  def get_all_docs()
+		@dbh.select_all("SELECT * FROM docs") do | row | 
+		  doc = Document.new(row['doctitle'])
+		  doc.dbid = row['docid']
+		end
+	  end
+	  # fetch the document identified by the string ident
+	  def get_doc(ident)
+		doc = nil
+		if ident =~ /^\d+/
+		  r = @dbh.select_one("SELECT doctitle FROM docs WHERE docid = ?", 
+							  ident)
+		  unless r
+			raise "No document found matching id '#{ident}'"
+		  end
+		  doc = Document.new(r[0])
+		  doc.dbid = ident
+		else
+		  r = @dbh.select_one("SELECT docid FROM docs WHERE doctitle = ?", ident)
+		  unless r
+			raise "No document found matching title '#{ident}'"
+		  end
+		  doc = Document.new(ident)
+		  doc.dbid = r[0]
+		end	  
+
+		text = ''
+		@dbh.select_all("SELECT * FROM chunks 
+		       		 WHERE docid = ?
+				     ORDER BY docseq", doc.dbid) do | row |
+		  doc.append(row['chunk'], row['type'])
+
+		end
+		doc
+	  end
+
+	  # decide what level to code at?
+	  # this returns a weighted search result set
+	  def doc_search_by_category(category)
+		# SELECT SUM(SQRT(idx_cat.score * idx_doc.score)), idx_doc.catid
+		# 	FROM reverse_index AS idx_doc, reverse_index AS idx_cat
+		# 	WHERE idx_cat.catid = 1
+		# 	AND idx_doc.catid != idx_cat.catid
+		# 	AND idx_doc.word = idx_cat.word
+		# 	GROUP BY idx_doc.catid;
+	  end
+
+	  def save_document(doc)
+		if doc.dbid
+		  @dbh.do("UPDATE docs SET doctitle = ? WHERE docid = ?", 
+				  doc.title, doc.dbid)
+		else
+		  @dbh.do("INSERT INTO docs VALUES(NULL, ?)", @title)
+		  r = @dbh.select_one("SELECT LAST_INSERT_ID()")
+		  doc.dbid = r[0]
+		  doc.fragments.each_with_index do | frag, i |
+			@dbh.do("INSERT INTO chunks VALUES(?, 0, ?, ?)", 
+					doc.dbid, frag.text, i)
+			doc.dbid = r[0]
+		  end
+		end
+		self
+	  end
+	  
+	  def connect(args)
+		@dbh = args[:dbh]
+	  end
+	  
+	  def get_category(catid)
+		r = @dbh.select_one("SELECT * FROM categories WHERE catid = ?", 
+							catid)
+		return nil unless r
+		# raise "No category found matching id '#{cat}'" unless r
+		category = Category.new(r['catname'])
+
+
+		# not found
+		category.dbid = catid
+
+		@dbh.select_all("SELECT codes.docid, codes.start, codes.offset
+                            FROM codes
+                            WHERE catid = ? ", category.dbid) do | row |
+		  category.code(row['docid'], row['start'].to_i, row['offset'].to_i)
+
+		end
+		category
+	  end
+
+	  def category_search_by_words(*words)
+		words = words.collect { | word | @dbh.quote(word) }
+		cats = []
+		@dbh.select_all("SELECT category.catid, category.catname, " +
+						"SUM(category_ridx.score) AS score " +
+						"FROM category_ridx, category " +
+						"WHERE category_ridx.catid =  category.catid " + 
+						"AND category_ridx.word IN ( ? ) " +
+						"GROUP BY category_ridx.catid", 
+						words.join(',') ) do | r |
+
+		  cat = Category.new(r['catname'])
+		  cat.dbid = r['catid']	  
+		  cats.push(cat)
+		end
+		cats
+	  end
+
+	  # fetches all the categories in a tree structure, starting from the right
+	  def get_all_categories()
+		parents = [ { 'cat' => Category.new('root'), 'r' => 10 } ]
+		l = 0 # avoid reinit
+		r = 0 # avoid reinit
+		cat = nil
+		
+		query = "SELECT * FROM categories ORDER BY l"
+		@dbh.select_all(query) do | row |
+		  l = row['l'].to_i
+		  r = row['r'].to_i
+		  cat = Category.new(row['catname'])
+		  cat.dbid = row['catid'].to_i
+		  # if a leaf
+		  if l + 1 == r
+			parents[-1]['cat'].add_child(cat)
+		  else
+			parents.push( { 'cat' => cat, 'r' => r } )
+		  end
+		  
+		  until parents.length == 1 || r <= parents[-1]['r']
+			parents.pop.fetch('cat').append_to(parents[-1]['cat'])
+		  end
+		end
+		
+		# clean up outstanding items to be added
+		while parents.length > 1
+		  parents.pop.fetch('cat').append_to(parents[-1]['cat'])
+		end
+		
+		return parents[0]['cat']
+	  end
+
+	  def save_category(cat)
+		if cat.dbid
+		  @dbh.do("DELETE FROM codes WHERE catid = ?", cat.dbid)
+		  @dbh.do("UPDATE categories SET catname = ? WHERE catid = ?",
+				  cat.name, cat.dbid)
+		else
+		  l = nil
+		  if cat.parent
+			row = @dbh.select_one("SELECT r FROM categories
+                             WHERE catid = ? ", cat.parent.dbid)
+			l = row['r'].to_i
+		  else
+			row = @dbh.select_one("SELECT MAX(r) r FROM categories")
+			l = row['r'].to_i + 1
+		  end
+		  puts "got l = #{l}"
+
+		  @dbh.do("INSERT INTO categories VALUES(NULL, ?, '', ?, ?)", 
+				  cat.name, l, l + 1)
+		  r = @dbh.select_one("SELECT LAST_INSERT_ID()")
+		  cat.dbid = r[0]
+		  @dbh.do("UPDATE categories SET l = l + 2
+               WHERE l > ? ", l)
+		  @dbh.do("UPDATE categories SET r = r + 2
+               WHERE r >= ? 
+               AND catid != ? ", l, cat.dbid) 
+		end
+
+		cat.vectors.each do | docid, vecs |
+		  vecs.each do | vec |
+			@dbh.do("INSERT INTO codes VALUES(?, ?, ?, ?)", 
+					cat.dbid, docid, vec.start, vec.length )
+		  end
+		end
+		
+		self
+	  end
+ 
+	  ## WARN - below here is older code imported from a different project
+	  ## NOT YET TESTED FOR COMPATIBILITY
+	  def save()
+		sql_install()
+		@docs.values.each do | doc |
+		  sql_store_document(doc)
+		end
+		i = 0
+		counter = proc { i += 1 }
+		
+		sql_store_node(@root_node, counter)
+	  end
+
+	  def sql_install()
+		@dbh.do("DROP TABLE IF EXISTS docs")
+		
+		# docid is just nudist's internal document keying system
+		@dbh.do("CREATE TABLE docs (
+			docid varchar(8) primary key,
+			doctitle varchar(255),
+			docmemo text,
+			external CHAR(1) )")
+		
+		@dbh.do("DROP TABLE IF EXISTS fragments")
+		@dbh.do("CREATE TABLE fragments (
+			docid VARCHAR(8) REFERENCES doc(docid),
+			seq INT,
+			fragment text)")
+
+		@dbh.do("DROP TABLE IF EXISTS nodes")
+		@dbh.do("CREATE TABLE nodes (
+			nodeid VARCHAR(32) PRIMARY KEY,
+			nodepath VARCHAR(255),
+			nodeuid VARCHAR(255),
+			nodename VARCHAR(32),
+			nodememo TEXT,
+			l INT,
+			r INT )")
+		
+		@dbh.do("DROP TABLE IF EXISTS vectors")
+		@dbh.do("CREATE TABLE vectors (
+			nodeid VARCHAR(32) REFERENCES nodes(nodeid),
+			docid VARCHAR(8) REFERENCES doc(docid),
+			start INT REFERENCES DOCFRAGMENT(seq),
+			length INT)")
+	  end
+	  
+	  def sql_store_node(node, counter)
+		left = counter.call()
+		node.children.each do | child |
+		  sql_store_node(child, counter)
+		end
+		right = counter.call()
+		
+		@dbh.do("INSERT INTO nodes VALUES(?, ?, ?, ?, ?, ?, ?)",
+				node.uniq_id, node.path, node.child_id, node.title,
+				node.memo, left, right)
+		node.codes.each do | doc, vectors |
+		  vectors.each do | vector |
+			@dbh.do("INSERT INTO vectors VALUES (?, ?, ?, ?)",
+					node.uniq_id, vector.doc.doc_id, 
+					vector.start, vector.length)
+		  end
+		end
+	  end
+	  
+	  def sql_store_document(doc)
+		@dbh.do("INSERT INTO docs VALUES (?, ?, ?, ?)",
+				doc.doc_id, doc.title, doc.memo, doc.external?)
+		
+		doc.fragments.each_with_index do | frag, i |
+		  @dbh.do("INSERT INTO fragments VALUES(?, ?, ?)",
+				  doc.doc_id, i, frag)
+		end
+	  end
+	end
+  end
+end