uhferret 1.3.7

data/README.rdoc

@@ -0,0 +1,79 @@
+= UHFerret 
+
+homepage:: https://peterlane.netlify.org/ferret/
+source:: https://notabug.org/peterlane/uhferret-gem/releases
+
+== Description
+
+UHFerret is a copy-detection tool, supporting the analysis of large sets of
+documents to find pairs of documents with substantial amounts of lexical
+copying. Documents containing either natural language (e.g. English) or
+computer programs (in C-family) may be processed.  
+
+This library provides a Ruby wrapper around uhferret suitable for 
+scripting, a command-line executable, 'uhferret', and a simple 
+server version, 'uhferret-server'.
+
+NB: to install uhferret, Ruby must be able to compile and build C extensions.
+
+== Use 
+
+=== Command Line
+
+    Usage: uhferret [options] file1 file2 ...
+        -h, --help                       help message
+        -c, --code                       process documents as code
+        -t, --text                       process documents as text (default)
+        -d, --data-table                 output similarity table (default)
+        -l, --list-trigrams              output trigram list
+        -a, --all-comparisons            output list of all comparisons
+        -x, --xml-report FILE            generate xml report from two documents
+        -f, --definition-file FILE       read document names from file
+
+To compute the similarities of a set of files, use:
+
+   $ uhferret file1.txt file2.txt ...
+
+An xml output can be generated for a pair of files using:
+
+   $ uhferret -x outfile.xml file1.txt file2.txt
+
+The xml output can be displayed in a browser using the style sheet 
+'uhferret.xsl' in the examples folder, and then printed from the browser.
+
+=== Program
+
+Ferret can also be used as a library, and called from within a program. 
+For example:
+
+  ferret = Ferret.new
+  ferret.add 'filename1.txt'
+  ferret.add 'filename2.txt'
+  ferret.run
+  ferret.output_similarity_table
+
+Will create a new instance of Ferret, add two documents, run and then output the 
+similarity between the two.
+
+=== Server
+
+    Usage: uhferret-server [options]
+        -h, --help                       help message
+        -p, --port n                     port number
+        -f, --folder FOLDER              base folder
+
+The folder to store the processed files will default to 
+'FerretFiles' and the port to 2000.
+Initial address: http://localhost:2000/ferret/home
+
+NB: The server uses some \*nix commands, and so currently does not work 
+under Windows.
+
+== Acknowledgements
+
+UHFerret has been developed at the University of Hertfordshire by members of
+the Plagiarism Detection Group.  The original concept of using trigrams for
+measuring copying was developed by Caroline Lyon and James Malcolm.  JunPeng
+Bao, Ruth Barrett and Bob Dickerson also contributed to the development of
+earlier versions of Ferret.
+

data/bin/uhferret

@@ -0,0 +1,129 @@
+#! /usr/bin/env ruby
+
+# This file is part of uhferret, providing a command-line interface.
+#
+# Author::    Peter Lane
+# Copyright:: Copyright 2012-20, Peter Lane.
+# License::   GPLv3
+#
+# uhferret is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# uhferret is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with uhferret.  If not, see <http://www.gnu.org/licenses/>.
+
+require "optparse"
+require "uhferret"
+
+MAX_TABLE_SIZE = 100
+VERSION = "1.3.7"
+
+# ------------------------------------------------------------------
+# 
+document_type = UHFerret::TextDocument
+input_format = :from_argv
+output_format = :similarity_table
+definitions_file = nil
+output_full_path = false
+xml_output_file = ""
+
+options = OptionParser.new do |opts|
+  opts.banner = "Usage: uhferret [options] file1 file2 ..."
+  opts.on("-h", "--help", "help message") do |v|
+    puts options
+    exit!
+  end
+  opts.on("-v", "--version", "version") do |v|
+    puts "uhferret: version #{VERSION}"
+    exit!
+  end
+  # -- document type
+  opts.on("-c", "--code", "process documents as code") do |v| 
+    document_type = UHFerret::CodeDocument
+  end
+  opts.on("-t", "--text", "process documents as text (default)") do |v| 
+    document_type = UHFerret::TextDocument
+  end
+  # -- output format
+  opts.on("-p", "--full-path", "output full path") do |v|
+    output_full_path = true
+  end
+  opts.on("-d", "--data-table", "output similarity table (default)") do |v|
+    output_format = :similarity_table
+  end
+  opts.on("-w", "--html-data-table", "output similarity table in html format") do |v|
+    output_format = :html_similarity_table
+  end
+  opts.on("-l", "--list-trigrams", "output trigram list") do |v|
+    output_format = :trigram_list
+  end
+  opts.on("-a", "--all-comparisons", "output list of all comparisons") do |v|
+    output_format = :all_comparisons
+  end
+  opts.on("-x", "--xml-report OUTPUT_FILE", "generate xml report from two documents") do |file|
+    output_format = :xml_output
+    xml_output_file = file
+  end
+  # -- file source
+  opts.on("-f FILE", "--definition-file FILE", "read document names from file") do |file|
+    input_format = :from_file
+    definitions_file = file
+  end
+end
+
+begin
+  # -- process input options
+  options.parse!
+
+  # -- check some errors
+  if output_format == :xml_output
+    unless ARGV.size == 2
+      puts "Error: for xml report, only provide two input filenames"
+      raise ArgumentError.new
+    end
+  end
+
+  # -- add readable files, and run
+  ferret = UHFerret::Ferret.new
+  unless definitions_file.nil?
+    if File.readable? definitions_file
+      ferret.add_list_from_file(definitions_file, document_type)
+    end
+  end
+  ARGV.each do |filename|
+    if File.readable? filename
+      ferret.add(filename, document_type)
+    end
+  end
+  if ferret.size < 2
+    puts "Error: not enough valid filenames"
+    raise ArgumentError.new
+  end
+  ferret.run
+
+  # -- display output
+  case output_format
+  when :similarity_table
+    ferret.output_similarity_table output_full_path
+  when :html_similarity_table
+    ferret.output_html_similarity_table
+  when :trigram_list
+    ferret.output_trigram_list
+  when :all_comparisons
+    ferret.output_all_comparisons
+  when :xml_output
+    ferret.xml_output(xml_output_file, 0, 1)
+  end
+
+rescue Exception => err
+  puts err
+  puts options
+end
+

data/bin/uhferret-server

@@ -0,0 +1,68 @@
+#! /usr/bin/env ruby
+#
+# This file is part of uhferret, providing a web service interface.
+#
+# Author::    Peter Lane
+# Copyright:: Copyright 2012, Peter Lane.
+# License::   GPLv3
+#
+# uhferret is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# uhferret is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with uhferret.  If not, see <http://www.gnu.org/licenses/>.
+
+require "optparse"
+require "webferret"
+
+# Name of the ferret software
+FERRET = "uhferret"
+# Base folder for working with files
+$base = "#{Dir.pwd}/FerretFiles/"
+# Port number for server
+port = 2000
+
+options = OptionParser.new do |opts|
+  opts.banner = "Usage: uhferret-server [options]"
+  opts.on("-h", "--help", "help message") do |v|
+    puts options
+    exit!
+  end
+  # -- port number
+  opts.on("-p", "--port n", Integer, "port number") do |v|
+    port = v
+  end
+  # -- base folder
+  opts.on("-f", "--folder FOLDER", "base folder") do |v|
+    $base = File.expand_path v
+  end
+end
+
+begin
+  options.parse!
+
+  # Refuse to work if the folder already exists
+  if File.exists? $base
+    puts "Folder #{$base} already exists"
+    puts "Please empty the folder before starting the ferret server"
+    exit!
+  end
+
+  # install the ferret server at given Port number
+  ferret = HTTPServer.new(:Port => port, :DocumentRoot => "/")
+
+  ferret.mount "/ferret/home", UHFerret::FerretHomeServlet
+  ferret.mount "/ferret/report", UHFerret::FerretReportServlet
+
+  trap("INT") { ferret.shutdown }
+
+  ferret.start
+end
+

data/ext/document.cpp

@@ -0,0 +1,231 @@
+#include "document.h"
+
+/** 
+ * This file is part of uhferret.
+ *
+ * Author::    Peter Lane
+ * Copyright:: Copyright 2011, Peter Lane.
+ * License::   GPLv3
+ *
+ * uhferret is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * uhferret is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with uhferret.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+Document::Document (std::string pathname, DocumentType type, int id)
+	: _pathname (pathname),
+	  _type (type),
+	  _num_trigrams (0),
+	  _group_id (id)
+{}
+
+Document::Document (Document * document)
+	: _pathname (document->_pathname),
+	  _type (document->_type),
+	  _num_trigrams (0),
+	  _group_id (document->_group_id)
+{}
+
+void Document::SetType (DocumentType type)
+{
+	_type = type;
+}
+
+bool Document::IsTextType () const
+{
+	return (_type == TypeText);
+}
+
+std::string Document::GetPathname () const 
+{
+	return _pathname;
+}
+
+void Document::SetPathname (std::string pathname)
+{
+	_pathname = pathname;
+}
+
+int Document::GetGroupId () const
+{
+	return _group_id;
+}
+
+void Document::SetGroupId (int id) 
+{
+	_group_id = id;
+}
+
+int Document::GetTrigramCount () const
+{
+	return _num_trigrams;
+}
+
+// WARNING: This method should only be used when document definitions are 
+// loaded from a file.
+void Document::SetTrigramCount (int count)
+{
+	_num_trigrams = count;
+}
+
+void Document::ResetTrigramCount ()
+{
+	_num_trigrams = 0;
+}
+
+void Document::IncrementTrigramCount ()
+{
+	_num_trigrams += 1;
+}
+
+// Start input from the file referred to by this document
+bool Document::StartInput (TokenSet & tokenset)
+{
+	_fb = new std::ifstream(GetPathname().c_str (), std::ifstream::in); 
+	if (_fb->is_open ())
+	{
+		_cin = _fb;
+		InitialiseInput (tokenset);
+		return true;	// signify file opened correctly
+	}
+	else
+	{
+		return false;
+	}
+}
+
+// Start input from a provided input stream
+bool Document::StartInput (std::istream & input, TokenSet & tokenset)
+{
+	_cin = &input;
+	InitialiseInput (tokenset);
+	return true;
+}
+
+// Start input by constructing a new Reader based on current document type
+// TokenSet is provided by caller, so Reader uses common set of labels for tokens
+void Document::InitialiseInput (TokenSet & tokenset)
+{
+	if (_type == TypeText)
+	{
+		_token_input = new WordReader (* _cin);
+	}
+	else // (_type == typeCode)
+	{
+		_token_input = new CCodeReader (* _cin);
+	}
+	ReadTrigram (tokenset); // read first two tokens so next call to 
+	ReadTrigram (tokenset); // ReadTrigram returns the first complete trigram
+}
+
+// returns true if this document's filetype is the same as the given extension
+// -- note, case is ignored, so "txt" == "TXT" == "tXt"
+bool Document::IsFileType (std::string extension) const
+{
+	int dot_posn = _pathname.find_last_of ('.', true); // search for last dot, i.e. from end
+	if (dot_posn == _pathname.npos) return false;
+	std::string file_extension = _pathname.substr (dot_posn+1, _pathname.npos-(dot_posn+1));
+
+	return StringToUpper(file_extension) == StringToUpper(extension); // ignore case in comparison
+}
+
+std::string Document::StringToUpper (std::string str) const
+{
+  std::string nstr = "";
+
+  for(int i=0, l = str.length (); i < l; i += 1)
+  {
+    nstr += std::toupper(str[i]);
+  }
+  return nstr;
+}
+
+// Test if file extension represents a c-type language
+bool Document::IsCodeType () const
+{
+	return  IsFileType ("cpp") ||
+		IsFileType ("c") ||
+		IsFileType ("java") ||
+		IsFileType ("h");
+}
+
+// Test if file extension represents a pure text document
+bool Document::IsTxtType () const
+{
+	return IsFileType ("txt");
+}
+
+// Test if file is not a known type
+bool Document::IsUnknownType () const
+{
+	return ! (IsCodeType () || IsTxtType ());
+}
+
+// Reads next input token and updates information held on current trigram.
+// return true if a trigram has been read and is ready for retrieval
+bool Document::ReadTrigram (TokenSet & tokenset)
+{
+	_current_tuple[0] = _current_tuple[1];
+	_current_tuple[1] = _current_tuple[2];
+	_current_start[0] = _current_start[1];
+	_current_start[1] = _current_start[2];
+	if ( _token_input->ReadToken () )
+	{
+		_current_tuple[2] = _token_input->GetToken (tokenset);
+		_current_start[2] = _token_input->GetTokenStart ();
+		return true;
+	}
+	else 
+	{
+		return false;
+	}
+	
+}
+
+// retrieve a token of the current tuple, based on position within tuple
+// -- as we only deal with trigrams, index must be in [0,2]
+std::size_t Document::GetToken (int i) const
+{
+	assert (i>=0 && i<=2);
+	return _current_tuple[i];
+}
+
+// retrieve the start position of current trigram
+std::size_t Document::GetTrigramStart () const
+{
+	return _current_start[0];
+}
+
+// retrieve the start position of token i within current trigram
+// -- used to get start position of second word
+std::size_t Document::GetTrigramStart (int i) const
+{
+	assert (i>=0 && i<=2);
+	return _current_start[i];
+}
+
+// retrieve the end position of the current token
+std::size_t Document::GetTrigramEnd () const
+{
+	return _token_input->GetTokenEnd ();
+}
+
+// Close up the input file buffer
+void Document::CloseInput ()
+{
+	delete _token_input;
+	if (_fb->is_open ())
+	{
+		_fb->close ();
+	}
+}
+