StrIdx 0.1.0 → 0.1.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 1079ebe4265ed15d2307d260395e33471ab149bc6cd33ca6e469ff93c960dba8
-  data.tar.gz: 0bdba2496690ad9e1794d653bb5ca047a15fcfee6d2988ff06098c69920ae269
+  metadata.gz: 2155c54900f595ae954d2074840f425195a66d6bf464e35dc68a1bb10236255c
+  data.tar.gz: fc2b50bcf6083b806643886a275e2a03ce3267cbd66dd81a0d63f5f68f8e4268
 SHA512:
-  metadata.gz: 8a1b2fe9ce6f87a8585f5b9ca86092f21c600575b704fc3bf4ebaa8ae3bd0897f7e272fb580fb98c38e8b65a5b95d0fd8cc1c3baeafdb235fe1d48e51153ae6c
-  data.tar.gz: b4a62b6251a38637905653f2aaa919653465fbc453844a2f02466ce413a8a1de0946de09973535472a70ccbc17151d426e76fe92b3338ccc0af9f83f6821b201
+  metadata.gz: ac27e251a448f1ca17d4672d12ac165412f2c7fbd71359e77d21f8a8c30521797befe7f2555d181be2af0cace345636558b4a8d93d8e141f707e8b28ffe5687b
+  data.tar.gz: f8e65efbed5079c0a2e37504aa955e0855d0ef088ad07d1d2566dee6c5f51b79d9ba71ef102e5aaeef8d1b7d548eefa2fab3930e21b7de05b940da882c9b86f3

data/Makefile

@@ -1,268 +1,9 @@
+all: demo
 
-SHELL = /bin/sh
+demo: *.hpp *.cpp
+	g++  -Wall -O3 -fopenmp -lstdc++ demo.cpp -o demo
+	
+clean: 
+	rm demo
 
-# V=0 quiet, V=1 verbose.  other values don't work.
-V = 0
-V0 = $(V:0=)
-Q1 = $(V:1=)
-Q = $(Q1:0=@)
-ECHO1 = $(V:1=@ :)
-ECHO = $(ECHO1:0=@ echo)
-NULLCMD = :
 
-#### Start of system configuration section. ####
-
-srcdir = .
-topdir = /home/samsam/.rbenv/versions/3.1.4/include/ruby-3.1.0
-hdrdir = $(topdir)
-arch_hdrdir = /home/samsam/.rbenv/versions/3.1.4/include/ruby-3.1.0/x86_64-linux
-PATH_SEPARATOR = :
-VPATH = $(srcdir):$(arch_hdrdir)/ruby:$(hdrdir)/ruby
-prefix = $(DESTDIR)/home/samsam/.rbenv/versions/3.1.4
-rubysitearchprefix = $(rubylibprefix)/$(sitearch)
-rubyarchprefix = $(rubylibprefix)/$(arch)
-rubylibprefix = $(libdir)/$(RUBY_BASE_NAME)
-exec_prefix = $(prefix)
-vendorarchhdrdir = $(vendorhdrdir)/$(sitearch)
-sitearchhdrdir = $(sitehdrdir)/$(sitearch)
-rubyarchhdrdir = $(rubyhdrdir)/$(arch)
-vendorhdrdir = $(rubyhdrdir)/vendor_ruby
-sitehdrdir = $(rubyhdrdir)/site_ruby
-rubyhdrdir = $(includedir)/$(RUBY_VERSION_NAME)
-vendorarchdir = $(vendorlibdir)/$(sitearch)
-vendorlibdir = $(vendordir)/$(ruby_version)
-vendordir = $(rubylibprefix)/vendor_ruby
-sitearchdir = $(sitelibdir)/$(sitearch)
-sitelibdir = $(sitedir)/$(ruby_version)
-sitedir = $(rubylibprefix)/site_ruby
-rubyarchdir = $(rubylibdir)/$(arch)
-rubylibdir = $(rubylibprefix)/$(ruby_version)
-sitearchincludedir = $(includedir)/$(sitearch)
-archincludedir = $(includedir)/$(arch)
-sitearchlibdir = $(libdir)/$(sitearch)
-archlibdir = $(libdir)/$(arch)
-ridir = $(datarootdir)/$(RI_BASE_NAME)
-mandir = $(datarootdir)/man
-localedir = $(datarootdir)/locale
-libdir = $(exec_prefix)/lib
-psdir = $(docdir)
-pdfdir = $(docdir)
-dvidir = $(docdir)
-htmldir = $(docdir)
-infodir = $(datarootdir)/info
-docdir = $(datarootdir)/doc/$(PACKAGE)
-oldincludedir = $(DESTDIR)/usr/include
-includedir = $(prefix)/include
-runstatedir = $(localstatedir)/run
-localstatedir = $(prefix)/var
-sharedstatedir = $(prefix)/com
-sysconfdir = $(prefix)/etc
-datadir = $(datarootdir)
-datarootdir = $(prefix)/share
-libexecdir = $(exec_prefix)/libexec
-sbindir = $(exec_prefix)/sbin
-bindir = $(exec_prefix)/bin
-archdir = $(rubyarchdir)
-
-
-CC_WRAPPER = 
-CC = gcc
-CXX = g++
-LIBRUBY = $(LIBRUBY_SO)
-LIBRUBY_A = lib$(RUBY_SO_NAME)-static.a
-LIBRUBYARG_SHARED = -Wl,-rpath,$(libdir) -L$(libdir) -l$(RUBY_SO_NAME)
-LIBRUBYARG_STATIC = -Wl,-rpath,$(libdir) -L$(libdir) -l$(RUBY_SO_NAME)-static $(MAINLIBS)
-empty =
-OUTFLAG = -o $(empty)
-COUTFLAG = -o $(empty)
-CSRCFLAG = $(empty)
-
-RUBY_EXTCONF_H = 
-cflags   = $(optflags) $(debugflags) $(warnflags)
-cxxflags = 
-optflags = -O3 -fno-fast-math
-debugflags = -ggdb3
-warnflags = -Wall -Wextra -Wdeprecated-declarations -Wduplicated-cond -Wimplicit-function-declaration -Wimplicit-int -Wmisleading-indentation -Wpointer-arith -Wwrite-strings -Wold-style-definition -Wimplicit-fallthrough=0 -Wmissing-noreturn -Wno-cast-function-type -Wno-constant-logical-operand -Wno-long-long -Wno-missing-field-initializers -Wno-overlength-strings -Wno-packed-bitfield-compat -Wno-parentheses-equality -Wno-self-assign -Wno-tautological-compare -Wno-unused-parameter -Wno-unused-value -Wsuggest-attribute=format -Wsuggest-attribute=noreturn -Wunused-variable -Wundef
-cppflags = 
-CCDLFLAGS = -fPIC
-CFLAGS   = $(CCDLFLAGS) $(cflags)  -fPIC $(ARCH_FLAG)
-INCFLAGS = -I. -I$(arch_hdrdir) -I$(hdrdir)/ruby/backward -I$(hdrdir) -I$(srcdir)
-DEFS     = 
-CPPFLAGS =  -I/home/samsam/.rbenv/versions/3.1.4/include  $(DEFS) $(cppflags)
-CXXFLAGS = $(CCDLFLAGS)  -Wall -Wno-unused-variable -O3 -fopenmp $(ARCH_FLAG)
-ldflags  = -L. -L/home/samsam/.rbenv/versions/3.1.4/lib  -fstack-protector-strong -rdynamic -Wl,-export-dynamic -Wl,--no-as-needed
-dldflags = -L/home/samsam/.rbenv/versions/3.1.4/lib  -Wl,--compress-debug-sections=zlib 
-ARCH_FLAG = 
-DLDFLAGS = $(ldflags) $(dldflags) $(ARCH_FLAG)
-LDSHARED = $(CC) -shared
-LDSHAREDXX = $(CXX) -shared
-AR = gcc-ar
-EXEEXT = 
-
-RUBY_INSTALL_NAME = $(RUBY_BASE_NAME)
-RUBY_SO_NAME = ruby
-RUBYW_INSTALL_NAME = 
-RUBY_VERSION_NAME = $(RUBY_BASE_NAME)-$(ruby_version)
-RUBYW_BASE_NAME = rubyw
-RUBY_BASE_NAME = ruby
-
-arch = x86_64-linux
-sitearch = $(arch)
-ruby_version = 3.1.0
-ruby = $(bindir)/$(RUBY_BASE_NAME)
-RUBY = $(ruby)
-BUILTRUBY = $(bindir)/$(RUBY_BASE_NAME)
-ruby_headers = $(hdrdir)/ruby.h $(hdrdir)/ruby/backward.h $(hdrdir)/ruby/ruby.h $(hdrdir)/ruby/defines.h $(hdrdir)/ruby/missing.h $(hdrdir)/ruby/intern.h $(hdrdir)/ruby/st.h $(hdrdir)/ruby/subst.h $(arch_hdrdir)/ruby/config.h
-
-RM = rm -f
-RM_RF = rm -fr
-RMDIRS = rmdir --ignore-fail-on-non-empty -p
-MAKEDIRS = /usr/bin/mkdir -p
-INSTALL = /usr/bin/install -c
-INSTALL_PROG = $(INSTALL) -m 0755
-INSTALL_DATA = $(INSTALL) -m 644
-COPY = cp
-TOUCH = exit >
-
-#### End of system configuration section. ####
-
-preload = 
-libpath = . $(libdir)
-LIBPATH =  -L. -L$(libdir) -Wl,-rpath,$(libdir)
-DEFFILE = 
-
-CLEANFILES = mkmf.log
-DISTCLEANFILES = 
-DISTCLEANDIRS = 
-
-extout = 
-extout_prefix = 
-target_prefix = 
-LOCAL_LIBS = 
-LIBS = $(LIBRUBYARG_SHARED) -lgomp -lstdc++  -lm  -lc
-ORIG_SRCS = demo.cpp ruby_interf.cpp
-SRCS = $(ORIG_SRCS) 
-OBJS = demo.o ruby_interf.o
-HDRS = $(srcdir)/unordered_dense.h $(srcdir)/stridx.hpp
-LOCAL_HDRS = 
-TARGET = stridx
-TARGET_NAME = stridx
-TARGET_ENTRY = Init_$(TARGET_NAME)
-DLLIB = $(TARGET).so
-EXTSTATIC = 
-STATIC_LIB = 
-
-TIMESTAMP_DIR = .
-BINDIR        = $(bindir)
-RUBYCOMMONDIR = $(sitedir)$(target_prefix)
-RUBYLIBDIR    = $(sitelibdir)$(target_prefix)
-RUBYARCHDIR   = $(sitearchdir)$(target_prefix)
-HDRDIR        = $(sitehdrdir)$(target_prefix)
-ARCHHDRDIR    = $(sitearchhdrdir)$(target_prefix)
-TARGET_SO_DIR =
-TARGET_SO     = $(TARGET_SO_DIR)$(DLLIB)
-CLEANLIBS     = $(TARGET_SO) false
-CLEANOBJS     = *.o  *.bak
-
-all:    $(DLLIB)
-static: $(STATIC_LIB)
-.PHONY: all install static install-so install-rb
-.PHONY: clean clean-so clean-static clean-rb
-
-clean-static::
-clean-rb-default::
-clean-rb::
-clean-so::
-clean: clean-so clean-static clean-rb-default clean-rb
-		-$(Q)$(RM_RF) $(CLEANLIBS) $(CLEANOBJS) $(CLEANFILES) .*.time
-
-distclean-rb-default::
-distclean-rb::
-distclean-so::
-distclean-static::
-distclean: clean distclean-so distclean-static distclean-rb-default distclean-rb
-		-$(Q)$(RM) Makefile $(RUBY_EXTCONF_H) conftest.* mkmf.log
-		-$(Q)$(RM) core ruby$(EXEEXT) *~ $(DISTCLEANFILES)
-		-$(Q)$(RMDIRS) $(DISTCLEANDIRS) 2> /dev/null || true
-
-realclean: distclean
-install: install-so install-rb
-
-install-so: $(DLLIB) $(TIMESTAMP_DIR)/.sitearchdir.time
-	$(INSTALL_PROG) $(DLLIB) $(RUBYARCHDIR)
-clean-static::
-	-$(Q)$(RM) $(STATIC_LIB)
-install-rb: pre-install-rb do-install-rb install-rb-default
-install-rb-default: pre-install-rb-default do-install-rb-default
-pre-install-rb: Makefile
-pre-install-rb-default: Makefile
-do-install-rb:
-do-install-rb-default:
-pre-install-rb-default:
-	@$(NULLCMD)
-$(TIMESTAMP_DIR)/.sitearchdir.time:
-	$(Q) $(MAKEDIRS) $(@D) $(RUBYARCHDIR)
-	$(Q) $(TOUCH) $@
-
-site-install: site-install-so site-install-rb
-site-install-so: install-so
-site-install-rb: install-rb
-
-.SUFFIXES: .c .m .cc .mm .cxx .cpp .o .S
-
-.cc.o:
-	$(ECHO) compiling $(<)
-	$(Q) $(CXX) $(INCFLAGS) $(CPPFLAGS) $(CXXFLAGS) $(COUTFLAG)$@ -c $(CSRCFLAG)$<
-
-.cc.S:
-	$(ECHO) translating $(<)
-	$(Q) $(CXX) $(INCFLAGS) $(CPPFLAGS) $(CXXFLAGS) $(COUTFLAG)$@ -S $(CSRCFLAG)$<
-
-.mm.o:
-	$(ECHO) compiling $(<)
-	$(Q) $(CXX) $(INCFLAGS) $(CPPFLAGS) $(CXXFLAGS) $(COUTFLAG)$@ -c $(CSRCFLAG)$<
-
-.mm.S:
-	$(ECHO) translating $(<)
-	$(Q) $(CXX) $(INCFLAGS) $(CPPFLAGS) $(CXXFLAGS) $(COUTFLAG)$@ -S $(CSRCFLAG)$<
-
-.cxx.o:
-	$(ECHO) compiling $(<)
-	$(Q) $(CXX) $(INCFLAGS) $(CPPFLAGS) $(CXXFLAGS) $(COUTFLAG)$@ -c $(CSRCFLAG)$<
-
-.cxx.S:
-	$(ECHO) translating $(<)
-	$(Q) $(CXX) $(INCFLAGS) $(CPPFLAGS) $(CXXFLAGS) $(COUTFLAG)$@ -S $(CSRCFLAG)$<
-
-.cpp.o:
-	$(ECHO) compiling $(<)
-	$(Q) $(CXX) $(INCFLAGS) $(CPPFLAGS) $(CXXFLAGS) $(COUTFLAG)$@ -c $(CSRCFLAG)$<
-
-.cpp.S:
-	$(ECHO) translating $(<)
-	$(Q) $(CXX) $(INCFLAGS) $(CPPFLAGS) $(CXXFLAGS) $(COUTFLAG)$@ -S $(CSRCFLAG)$<
-
-.c.o:
-	$(ECHO) compiling $(<)
-	$(Q) $(CC) $(INCFLAGS) $(CPPFLAGS) $(CFLAGS) $(COUTFLAG)$@ -c $(CSRCFLAG)$<
-
-.c.S:
-	$(ECHO) translating $(<)
-	$(Q) $(CC) $(INCFLAGS) $(CPPFLAGS) $(CFLAGS) $(COUTFLAG)$@ -S $(CSRCFLAG)$<
-
-.m.o:
-	$(ECHO) compiling $(<)
-	$(Q) $(CC) $(INCFLAGS) $(CPPFLAGS) $(CFLAGS) $(COUTFLAG)$@ -c $(CSRCFLAG)$<
-
-.m.S:
-	$(ECHO) translating $(<)
-	$(Q) $(CC) $(INCFLAGS) $(CPPFLAGS) $(CFLAGS) $(COUTFLAG)$@ -S $(CSRCFLAG)$<
-
-$(TARGET_SO): $(OBJS) Makefile
-	$(ECHO) linking shared-object $(DLLIB)
-	-$(Q)$(RM) $(@)
-	$(Q) $(LDSHAREDXX) -o $@ $(OBJS) $(LIBPATH) $(DLDFLAGS) $(LOCAL_LIBS) $(LIBS)
-
-
-
-$(OBJS): $(HDRS) $(ruby_headers)

data/README.md

@@ -1,4 +1,4 @@
-# stridx
+# StrIdx
 This library provides fast fuzzy string similarity search and indexing. It has been mainly developed for indexing filepaths, but can be used for other types of strings aswell. It can easily handle fuzzy searches for more than 100,000 filepaths.
 
 The fuzziness means that candidate filepaths do not need to include exact match of the query string. They are considered a good match if they include parts of the query string, and even if those parts are in the wrong order.
@@ -26,7 +26,7 @@ Sum up the scores for each character c and divide by (string length)^2
 For example, if query = "rngnomadriv" 
 and candidate is "./drivers/char/hw_random/nomadik-rng.c", then scores are calculated as follows:
 ```
-    rngnomadriv
+    rngnomadriv (substrings rng=3, nomad=5 and driv=4)
     33355555444 (subscores)
     FFFFFFFFDDD (F=file component, D=dir component)
     score1=(3+3+3+5+5+5+5+5+(4+4+4)*0.7)
@@ -36,6 +36,75 @@ and candidate is "./drivers/char/hw_random/nomadik-rng.c", then scores are calcu
     and minor part by (query string length)*(candidate string length)
     score = score1/(11*11)*0.97 + score1/(11*38)*0.03 = 0.342944
 ```
+
+# Ruby interface
+Install:
+```
+gem install StrIdx
+```
+
+Usage example (see test.rb):
+```ruby
+require "stridx"
+idx = StrIdx::StringIndex.new
+
+t = Time.new
+fn = File.expand_path("flist.txt")
+lines = IO.read(fn).lines.collect { |x| x.strip }
+i = 0
+for x in lines
+  idx.add(x, i)
+  i += 1
+end
+
+idx_time = Time.new 
+puts "\nIndexing time (#{lines.size} files}): #{(idx_time - t).round(4)} seconds"
+
+query = "rngnomadriv"
+res = idx.find(query)
+puts "query: #{query}"
+puts "\nResults:"
+puts "Filename, score"
+puts "==============="
+for id, score in res
+  fn = lines[id]
+  puts "#{fn}, #{score.round(4)}"
+end
+
+query_time = Time.new 
+
+puts "\nSearch time: #{(query_time - idx_time).round(4)} seconds"
+
+```
+
+Output:
+```
+Indexing time (89828 files}): 2.813722207
+query: rngnomadriv
+
+Results:
+Filename, score
+===============
+./drivers/char/hw_random/nomadik-rng.c, 0.3429
+./drivers/pinctrl/nomadik, 0.2714
+./drivers/clk/clk-nomadik.c, 0.2711
+./drivers/gpio/gpio-nomadik.c, 0.2709
+./drivers/i2c/busses/i2c-nomadik.c, 0.2704
+./drivers/clocksource/nomadik-mtu.c, 0.2704
+./drivers/gpu/drm/pl111/pl111_nomadik.h, 0.2701
+./drivers/gpu/drm/pl111/pl111_nomadik.c, 0.2701
+./drivers/pinctrl/nomadik/pinctrl-nomadik.c, 0.2699
+./drivers/input/keyboard/nomadik-ske-keypad.c, 0.2698
+./drivers/pinctrl/nomadik/pinctrl-nomadik-db8500.c, 0.2696
+./drivers/pinctrl/nomadik/pinctrl-nomadik-stn8815.c, 0.2695
+./drivers/char/hw_random/omap-rng.c, 0.2364
+./drivers/char/hw_random/omap3-rom-rng.c, 0.2361
+./include/dt-bindings/pinctrl/nomadik.h, 0.2248
+
+Search time: 0.0488 seconds
+```
+
+
 # C++ API
 See demo.cpp
 ```cpp