nysol-mining 3.0.0

data/ext/sketchsortrun/SketchSort.hpp

@@ -0,0 +1,138 @@
+/*
+ * SketchSort.hpp
+ * Copyright (c) 2011 Yasuo Tabei All Rights Reserved.
+ *
+ * Permission is hereby granted, free of charge, to any person
+ * obtaining a copy of this software and associated documentation
+ * files (the "Software"), to deal in the Software without
+ * restriction, including without limitation the rights to use,
+ * copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the * Software is furnished to do so, subject to the following
+ * conditions:
+ *
+ * The above copyright notice and this permission notice shall be * included in all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
+ * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE and * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
+ * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+ * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+#ifndef SKETCHSORT_HPP
+#define SKETCHSORT_HPP
+
+// for boost
+#include <boost/pool/pool.hpp>
+#include <boost/pool/object_pool.hpp>
+#include <boost/numeric/ublas/matrix.hpp>
+#include <boost/numeric/ublas/triangular.hpp>
+#include <boost/numeric/ublas/symmetric.hpp>
+#include <boost/numeric/ublas/hermitian.hpp>
+#include <boost/numeric/ublas/matrix_sparse.hpp>
+#include <boost/numeric/ublas/vector.hpp>
+#include <boost/numeric/ublas/vector_sparse.hpp>
+
+#include <iostream>
+#include <string>
+#include <vector>
+#include <list>
+#include <map>
+#include <cmath>
+#include <cstring>
+#include <iterator>
+#include <fstream>
+#include <strstream>
+#include <sstream>
+#include <cstdio>
+#include <cstdlib>
+#include <set>
+#include <stdint.h>
+#include <time.h>
+
+#include <boost/random.hpp>
+
+using namespace boost::numeric::ublas;  // boost::numeric::ublas
+using namespace boost::numeric;         // boost::numeric
+
+#ifndef M_PI
+#define M_PI 3.14159265358979323846/* pi */
+#define M_PI_2 1.57079632679489661923/* pi/2 */
+#define M_PI_4 0.78539816339744830962/* pi/4 */
+#endif
+
+struct pstat {
+  int start;
+  int end;
+};
+
+struct params {
+  unsigned int numblocks;
+  unsigned int numchunks;
+  unsigned int projectDim;
+  unsigned int chunk_dist;
+  unsigned int chunks;
+  unsigned int num_seq;
+  unsigned int seq_len;
+  unsigned int chunk_len;
+  unsigned int start_chunk;
+  unsigned int end_chunk;
+  unsigned int cchunk;
+  unsigned int *counter;
+  pstat        *pos;
+  pstat        *pchunks;
+  float         cosDist;
+  std::vector<unsigned int> ids;
+  std::vector<int> blocks;
+  std::ostream *os;
+};
+
+class SketchSort {
+  boost::pool<> *p;
+  std::vector<boost::numeric::ublas::vector<float> > fvs;
+	std::vector<int> tws;
+  std::vector<float> norms;
+  uint8_t num_char;
+  unsigned int dim;
+  //unsigned int tw;
+
+  uint64_t numSort;
+  uint64_t numHamDist;
+  uint64_t numCosDist;
+
+  void readFeature(const char *fname, unsigned int _windowsize);
+  void centeringData();
+  void preComputeNorms();
+  int projectVectors(unsigned int projectDim, std::vector<uint8_t*> &sig, unsigned int seed, params &param);
+  void report(std::vector<uint8_t*> &sig, int l, int r, params &param, unsigned int _windowsize);
+  void sort(std::vector<uint8_t*> &sig, int spos, int epos, int l, int r, params &param);
+  void radixsort(std::vector<uint8_t*> &sig, int spos, int epos, int l, int r, params &param);
+  void insertionSort(std::vector<uint8_t*> &sig, int spos, int epos, int l, int r, params &param);
+  bool calc_chunk_hamdist(uint8_t *seq1, uint8_t *seq2, const params &param);
+  bool calc_hamdist(uint8_t *seq1, uint8_t *seq2, const params &param);
+  bool check_canonical(uint8_t *seq1, uint8_t *seq2, const params &param);
+  bool check_chunk_canonical(uint8_t *seq1, uint8_t *seq2, const params &param);
+  float checkCos(unsigned int id1, unsigned int id2);
+  double calcMissingEdgeRatio(params &param);
+  void classify(std::vector<uint8_t*> &sig, int spos, int epos, int l, int r, int bpos, params &param, unsigned int _windowsize);
+  void multi_classification(std::vector<uint8_t*> &sig, int maxind, int l, int r, params &param, unsigned int _windowsize);
+  void refinement();
+  void insertKnnList(unsigned int from_id, unsigned int to_id, float cosDist);
+  void decideParameters(float _missingratio, params &param);
+ public:
+  SketchSort() {};
+  void run(const char *fname, const char *oname,
+	   unsigned int _numblocks, 
+           unsigned int _dist,
+	   float        _cosDist,
+	   unsigned int _numchunks, 
+	   bool         _autoFlag,
+	   float        _missingratio, 
+	   bool         _centering,
+	   unsigned int _windowsize,
+	   unsigned int _seed);
+};
+
+#endif // SKETCHSORT_HPP

data/ext/sketchsortrun/extconf.rb

@@ -0,0 +1,26 @@
+require "rubygems"
+require "mkmf"
+
+unless have_library("boost_system")
+  puts("need boost_system.")
+  exit 1
+end
+
+unless have_library("kgmod3")
+  puts("need libkgmod.")
+  puts("refer https://github.com/nysol/mcmd")
+  exit 1
+end
+
+
+
+cp = "$(srcdir)"
+
+$CFLAGS = " -O3 -DNDEBUG -D_NO_MAIN_ -Wno-deprecated -pedantic -ansi -finline-functions -foptimize-sibling-calls -Wcast-qual -Wwrite-strings -Wsign-promo -Wcast-align -Wno-long-long -fexpensive-optimizations -funroll-all-loops -ffast-math -fomit-frame-pointer -pipe -I./"
+$CPPFLAGS = " -O3 -DNDEBUG -D_NO_MAIN_ -Wno-deprecated -pedantic -ansi -finline-functions -foptimize-sibling-calls -Wcast-qual -Wwrite-strings -Wsign-promo -Wcast-align -Wno-long-long -fexpensive-optimizations -funroll-all-loops -ffast-math -fomit-frame-pointer -pipe -I./"
+$CXXFLAGS = " -O3 -DNDEBUG -D_NO_MAIN_ -Wno-deprecated -pedantic -ansi -finline-functions -foptimize-sibling-calls -Wcast-qual -Wwrite-strings -Wsign-promo -Wcast-align -Wno-long-long -fexpensive-optimizations -funroll-all-loops -ffast-math -fomit-frame-pointer -pipe -I./"
+
+$LOCAL_LIBS += " -lstdc++ -lkgmod3 -lm -lboost_system"
+
+create_makefile("nysol/sketchsortrun")
+

data/ext/sketchsortrun/sketchsortrun.cpp

@@ -0,0 +1,56 @@
+#include <stdio.h>
+#include <stdlib.h>
+#include <unistd.h>
+#include <fcntl.h>
+#include <sys/stat.h>
+#include <string>
+#include <ruby.h>
+
+#include "Main.hpp"
+
+
+#include <kgMethod.h>
+
+extern "C" {
+	void Init_sketchsortrun(void);
+}
+
+
+VALUE lcmrun(VALUE self,VALUE argvV){
+
+	string argstr=RSTRING_PTR(argvV);
+	vector<char *> opts = kglib::splitToken(const_cast<char*>(argstr.c_str()), ' ',true);
+
+	// 引数文字列へのポインタの領域はここでauto変数に確保する
+	kglib::kgAutoPtr2<char*> argv;
+	char** vv;
+	try{
+		argv.set(new char*[opts.size()+1]);
+		vv = argv.get();
+	}catch(...){
+		rb_raise(rb_eRuntimeError,"memory allocation error");
+	}
+
+	// vv配列0番目はコマンド名
+	vv[0]=const_cast<char*>("lcm");
+
+	size_t vvSize;
+	for(vvSize=0; vvSize<opts.size(); vvSize++){
+		vv[vvSize+1] = opts.at(vvSize);
+	}
+	vvSize+=1;
+	int rtn = sketchsort_main (vvSize,vv);
+	return INT2NUM(rtn);
+}
+
+// -----------------------------------------------------------------------------
+// ruby Mcsvin クラス init
+// -----------------------------------------------------------------------------
+void Init_sketchsortrun(void) 
+{
+	// モジュール定義:MCMD::xxxxの部分
+	VALUE mtake=rb_define_module("NYSOL_MINING");
+	rb_define_module_function(mtake,"run_sketchsort"       , (VALUE (*)(...))lcmrun,1);
+}
+
+

data/lib/nysol/mining.rb

@@ -0,0 +1,24 @@
+#!/usr/bin/env ruby
+#/* ////////// LICENSE INFO ////////////////////
+#
+# * Copyright (C) 2013 by NYSOL CORPORATION
+# *
+# * Unless you have received this program directly from NYSOL pursuant
+# * to the terms of a commercial license agreement with NYSOL, then
+# * this program is licensed to you under the terms of the GNU Affero General
+# * Public License (AGPL) as published by the Free Software Foundation,
+# * either version 3 of the License, or (at your option) any later version.
+# * 
+# * This program is distributed in the hope that it will be useful, but
+# * WITHOUT ANY EXPRESS OR IMPLIED WARRANTY, INCLUDING THOSE OF 
+# * NON-INFRINGEMENT, MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE.
+# *
+# * Please refer to the AGPL (http://www.gnu.org/licenses/agpl-3.0.txt)
+# * for more details.
+#
+# ////////// LICENSE INFO ////////////////////*/
+
+
+require "nysol/sketchsortrun"
+
+

metadata

@@ -0,0 +1,89 @@
+--- !ruby/object:Gem::Specification
+name: nysol-mining
+version: !ruby/object:Gem::Version
+  version: 3.0.0
+platform: ruby
+authors:
+- NYSOL
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2017-07-07 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: nysol
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 3.0.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 3.0.0
+description: |2
+      nysol Mining tools
+email: info@nysol.jp
+executables:
+- mburst.rb
+- mgpmetis.rb
+- mnb.rb
+- mbopt.rb
+- mnetsimile.rb
+- mgfeatures.rb
+- msketchsort.rb
+- mnewman.rb
+- mgnfeatures.rb
+- mglmnet.rb
+- msm.rb
+- midxmine.rb
+extensions:
+- ext/sketchsortrun/extconf.rb
+extra_rdoc_files: []
+files:
+- ext/sketchsortrun/extconf.rb
+- ext/sketchsortrun/Main.cpp
+- ext/sketchsortrun/Main.hpp
+- ext/sketchsortrun/SketchSort.cpp
+- ext/sketchsortrun/SketchSort.hpp
+- ext/sketchsortrun/sketchsortrun.cpp
+- lib/nysol/mining.rb
+- bin/mnb.rb
+- bin/mbopt.rb
+- bin/mburst.rb
+- bin/mgpmetis.rb
+- bin/mnetsimile.rb
+- bin/mgfeatures.rb
+- bin/msketchsort.rb
+- bin/mnewman.rb
+- bin/mgnfeatures.rb
+- bin/msm.rb
+- bin/mglmnet.rb
+- bin/midxmine.rb
+homepage: http://www.nysol.jp/
+licenses: []
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.0.14
+signing_key: 
+specification_version: 4
+summary: nysol mining tools
+test_files: []