mhs-xapian 1.0.18a

Sign up to get free protection for your applications and to get access to all the features.
Files changed (328) hide show
  1. data/.gitignore +3 -0
  2. data/AUTHORS +1 -0
  3. data/COPYING +340 -0
  4. data/ChangeLog +5876 -0
  5. data/HACKING +101 -0
  6. data/INSTALL +293 -0
  7. data/Makefile +722 -0
  8. data/Makefile.am +26 -0
  9. data/Makefile.in +722 -0
  10. data/NEWS +2110 -0
  11. data/README +59 -0
  12. data/Rakefile +51 -0
  13. data/TODO +47 -0
  14. data/aclocal.m4 +7675 -0
  15. data/config.guess +1501 -0
  16. data/config.h +56 -0
  17. data/config.h.in +55 -0
  18. data/config.status +1298 -0
  19. data/config.sub +1705 -0
  20. data/configure +18536 -0
  21. data/configure.ac +944 -0
  22. data/csharp/.deps/xapian_wrap.Plo +1 -0
  23. data/csharp/AssemblyInfo.cs +40 -0
  24. data/csharp/AssemblyInfo.cs.in +40 -0
  25. data/csharp/Auto.cs +46 -0
  26. data/csharp/BM25Weight.cs +107 -0
  27. data/csharp/BoolWeight.cs +103 -0
  28. data/csharp/Database.cs +275 -0
  29. data/csharp/DateValueRangeProcessor.cs +61 -0
  30. data/csharp/Document.cs +177 -0
  31. data/csharp/ESet.cs +94 -0
  32. data/csharp/ESetIterator.cs +117 -0
  33. data/csharp/Enquire.cs +274 -0
  34. data/csharp/ExpandDecider.cs +76 -0
  35. data/csharp/Flint.cs +58 -0
  36. data/csharp/InMemory.cs +46 -0
  37. data/csharp/MSet.cs +193 -0
  38. data/csharp/MSetIterator.cs +141 -0
  39. data/csharp/Makefile +868 -0
  40. data/csharp/Makefile.am +106 -0
  41. data/csharp/Makefile.in +868 -0
  42. data/csharp/MatchDecider.cs +76 -0
  43. data/csharp/MultiValueSorter.cs +63 -0
  44. data/csharp/NumberValueRangeProcessor.cs +61 -0
  45. data/csharp/PositionIterator.cs +101 -0
  46. data/csharp/PostingIterator.cs +125 -0
  47. data/csharp/Quartz.cs +58 -0
  48. data/csharp/Query.cs +150 -0
  49. data/csharp/QueryParser.cs +174 -0
  50. data/csharp/RSet.cs +102 -0
  51. data/csharp/Remote.cs +100 -0
  52. data/csharp/SWIGTYPE_p_std__string.cs +30 -0
  53. data/csharp/SWIGTYPE_p_std__vectorTXapian__Query_t.cs +30 -0
  54. data/csharp/SWIGTYPE_p_std__vectorTstd__string_t.cs +30 -0
  55. data/csharp/SimpleStopper.cs +64 -0
  56. data/csharp/SmokeTest.cs +178 -0
  57. data/csharp/Sorter.cs +76 -0
  58. data/csharp/Stem.cs +66 -0
  59. data/csharp/Stopper.cs +91 -0
  60. data/csharp/StringValueRangeProcessor.cs +53 -0
  61. data/csharp/TermGenerator.cs +152 -0
  62. data/csharp/TermIterator.cs +125 -0
  63. data/csharp/TradWeight.cs +107 -0
  64. data/csharp/ValueIterator.cs +102 -0
  65. data/csharp/ValueRangeProcessor.cs +76 -0
  66. data/csharp/Version.cs +60 -0
  67. data/csharp/Weight.cs +93 -0
  68. data/csharp/WritableDatabase.cs +153 -0
  69. data/csharp/Xapian.cs +65 -0
  70. data/csharp/XapianPINVOKE.cs +1527 -0
  71. data/csharp/docs/Makefile +450 -0
  72. data/csharp/docs/Makefile.am +16 -0
  73. data/csharp/docs/Makefile.in +450 -0
  74. data/csharp/docs/examples/SimpleExpand.cs +109 -0
  75. data/csharp/docs/examples/SimpleIndex.cs +71 -0
  76. data/csharp/docs/examples/SimpleSearch.cs +78 -0
  77. data/csharp/docs/index.html +211 -0
  78. data/csharp/util.i +233 -0
  79. data/csharp/xapian_wrap.cc +10338 -0
  80. data/csharp/xapian_wrap.h +93 -0
  81. data/depcomp +632 -0
  82. data/extconf.rb +20 -0
  83. data/generic/except.i +80 -0
  84. data/generic/generic.mk +48 -0
  85. data/install-sh +520 -0
  86. data/java-swig/.deps/xapian_wrap.Plo +1 -0
  87. data/java-swig/Auto.java +35 -0
  88. data/java-swig/BM25Weight.java +81 -0
  89. data/java-swig/BoolWeight.java +77 -0
  90. data/java-swig/Database.java +195 -0
  91. data/java-swig/DateValueRangeProcessor.java +51 -0
  92. data/java-swig/Document.java +135 -0
  93. data/java-swig/ESet.java +71 -0
  94. data/java-swig/ESetIterator.java +71 -0
  95. data/java-swig/Enquire.java +246 -0
  96. data/java-swig/ExpandDecider.java +59 -0
  97. data/java-swig/Flint.java +43 -0
  98. data/java-swig/InMemory.java +35 -0
  99. data/java-swig/MSet.java +143 -0
  100. data/java-swig/MSetIterator.java +87 -0
  101. data/java-swig/Makefile +781 -0
  102. data/java-swig/Makefile.am +132 -0
  103. data/java-swig/Makefile.in +781 -0
  104. data/java-swig/MatchDecider.java +59 -0
  105. data/java-swig/MultiValueSorter.java +51 -0
  106. data/java-swig/NumberValueRangeProcessor.java +51 -0
  107. data/java-swig/PositionIterator.java +63 -0
  108. data/java-swig/PostingIterator.java +83 -0
  109. data/java-swig/Quartz.java +43 -0
  110. data/java-swig/Query.java +189 -0
  111. data/java-swig/QueryParser.java +214 -0
  112. data/java-swig/RSet.java +79 -0
  113. data/java-swig/Remote.java +71 -0
  114. data/java-swig/SWIGTYPE_p_std__string.java +25 -0
  115. data/java-swig/SimpleStopper.java +51 -0
  116. data/java-swig/SmokeTest.java +161 -0
  117. data/java-swig/Sorter.java +59 -0
  118. data/java-swig/Stem.java +51 -0
  119. data/java-swig/Stopper.java +63 -0
  120. data/java-swig/StringValueRangeProcessor.java +43 -0
  121. data/java-swig/TermGenerator.java +158 -0
  122. data/java-swig/TermIterator.java +83 -0
  123. data/java-swig/TradWeight.java +81 -0
  124. data/java-swig/ValueIterator.java +67 -0
  125. data/java-swig/ValueRangeProcessor.java +59 -0
  126. data/java-swig/Version.java +47 -0
  127. data/java-swig/Weight.java +68 -0
  128. data/java-swig/WritableDatabase.java +123 -0
  129. data/java-swig/Xapian.java +39 -0
  130. data/java-swig/XapianConstants.java +15 -0
  131. data/java-swig/XapianJNI.java +508 -0
  132. data/java-swig/run-java-test +6 -0
  133. data/java-swig/xapian_wrap.cc +12594 -0
  134. data/java-swig/xapian_wrap.h +91 -0
  135. data/java/Makefile +660 -0
  136. data/java/Makefile.am +35 -0
  137. data/java/Makefile.in +660 -0
  138. data/java/README +76 -0
  139. data/java/SmokeTest.java +148 -0
  140. data/java/native/.deps/Database.Plo +1 -0
  141. data/java/native/.deps/Document.Plo +1 -0
  142. data/java/native/.deps/ESet.Plo +1 -0
  143. data/java/native/.deps/ESetIterator.Plo +1 -0
  144. data/java/native/.deps/Enquire.Plo +1 -0
  145. data/java/native/.deps/MSet.Plo +1 -0
  146. data/java/native/.deps/MSetIterator.Plo +1 -0
  147. data/java/native/.deps/PositionIterator.Plo +1 -0
  148. data/java/native/.deps/Query.Plo +1 -0
  149. data/java/native/.deps/RSet.Plo +1 -0
  150. data/java/native/.deps/Stem.Plo +1 -0
  151. data/java/native/.deps/TermIterator.Plo +1 -0
  152. data/java/native/.deps/WritableDatabase.Plo +1 -0
  153. data/java/native/.deps/org_xapian_XapianJNI.Plo +1 -0
  154. data/java/native/.deps/utils.Plo +1 -0
  155. data/java/native/Database.cc +222 -0
  156. data/java/native/Document.cc +173 -0
  157. data/java/native/ESet.cc +79 -0
  158. data/java/native/ESetIterator.cc +82 -0
  159. data/java/native/Enquire.cc +271 -0
  160. data/java/native/MSet.cc +169 -0
  161. data/java/native/MSetIterator.cc +107 -0
  162. data/java/native/Makefile +594 -0
  163. data/java/native/Makefile.am +51 -0
  164. data/java/native/Makefile.in +594 -0
  165. data/java/native/PositionIterator.cc +64 -0
  166. data/java/native/Query.cc +180 -0
  167. data/java/native/RSet.cc +98 -0
  168. data/java/native/Stem.cc +75 -0
  169. data/java/native/TermIterator.cc +107 -0
  170. data/java/native/WritableDatabase.cc +118 -0
  171. data/java/native/XapianObjectHolder.h +115 -0
  172. data/java/native/org_xapian_XapianJNI.cc +78 -0
  173. data/java/native/org_xapian_XapianJNI.h +1369 -0
  174. data/java/native/utils.cc +51 -0
  175. data/java/native/xapian_jni.h +116 -0
  176. data/java/org/xapian/Database.java +148 -0
  177. data/java/org/xapian/Document.java +135 -0
  178. data/java/org/xapian/ESet.java +66 -0
  179. data/java/org/xapian/ESetIterator.java +97 -0
  180. data/java/org/xapian/Enquire.java +136 -0
  181. data/java/org/xapian/ExpandDecider.java +30 -0
  182. data/java/org/xapian/MSet.java +104 -0
  183. data/java/org/xapian/MSetIterator.java +132 -0
  184. data/java/org/xapian/Makefile +580 -0
  185. data/java/org/xapian/Makefile.am +38 -0
  186. data/java/org/xapian/Makefile.in +580 -0
  187. data/java/org/xapian/MatchDecider.java +30 -0
  188. data/java/org/xapian/PositionIterator.java +89 -0
  189. data/java/org/xapian/Query.java +190 -0
  190. data/java/org/xapian/RSet.java +89 -0
  191. data/java/org/xapian/Stem.java +80 -0
  192. data/java/org/xapian/TermIterator.java +142 -0
  193. data/java/org/xapian/WritableDatabase.java +92 -0
  194. data/java/org/xapian/Xapian.java +114 -0
  195. data/java/org/xapian/XapianJNI.java +444 -0
  196. data/java/org/xapian/errors/AssertionError.java +40 -0
  197. data/java/org/xapian/errors/DatabaseCorruptError.java +40 -0
  198. data/java/org/xapian/errors/DatabaseError.java +40 -0
  199. data/java/org/xapian/errors/DatabaseLockError.java +40 -0
  200. data/java/org/xapian/errors/DatabaseModifiedError.java +40 -0
  201. data/java/org/xapian/errors/DatabaseOpeningError.java +40 -0
  202. data/java/org/xapian/errors/DocNotFoundError.java +40 -0
  203. data/java/org/xapian/errors/FeatureUnavailableError.java +40 -0
  204. data/java/org/xapian/errors/InternalError.java +40 -0
  205. data/java/org/xapian/errors/InvalidArgumentError.java +40 -0
  206. data/java/org/xapian/errors/InvalidOperationError.java +40 -0
  207. data/java/org/xapian/errors/LogicError.java +40 -0
  208. data/java/org/xapian/errors/Makefile +416 -0
  209. data/java/org/xapian/errors/Makefile.am +32 -0
  210. data/java/org/xapian/errors/Makefile.in +416 -0
  211. data/java/org/xapian/errors/NetworkError.java +40 -0
  212. data/java/org/xapian/errors/NetworkTimeoutError.java +40 -0
  213. data/java/org/xapian/errors/RangeError.java +40 -0
  214. data/java/org/xapian/errors/RuntimeError.java +40 -0
  215. data/java/org/xapian/errors/UnimplementedError.java +40 -0
  216. data/java/org/xapian/errors/XapianError.java +40 -0
  217. data/java/org/xapian/errors/XapianRuntimeError.java +49 -0
  218. data/java/org/xapian/examples/Makefile +391 -0
  219. data/java/org/xapian/examples/Makefile.am +8 -0
  220. data/java/org/xapian/examples/Makefile.in +391 -0
  221. data/java/org/xapian/examples/SimpleIndex.java +68 -0
  222. data/java/org/xapian/examples/SimpleSearch.java +71 -0
  223. data/java/run-java-test +6 -0
  224. data/libtool +7618 -0
  225. data/ltmain.sh +6956 -0
  226. data/mhs-xapian.gemspec +368 -0
  227. data/missing +378 -0
  228. data/php/.deps/xapian_wrap.Plo +1 -0
  229. data/php/Makefile +871 -0
  230. data/php/Makefile.am +82 -0
  231. data/php/Makefile.in +871 -0
  232. data/php/docs/Makefile +453 -0
  233. data/php/docs/Makefile.am +19 -0
  234. data/php/docs/Makefile.in +453 -0
  235. data/php/docs/examples/simpleexpand.php4 +108 -0
  236. data/php/docs/examples/simpleexpand.php5 +104 -0
  237. data/php/docs/examples/simpleindex.php4 +76 -0
  238. data/php/docs/examples/simpleindex.php5 +73 -0
  239. data/php/docs/examples/simplesearch.php4 +75 -0
  240. data/php/docs/examples/simplesearch.php5 +72 -0
  241. data/php/docs/index.html +313 -0
  242. data/php/except.i +98 -0
  243. data/php/php4/php_xapian.h +323 -0
  244. data/php/php4/xapian.php +32 -0
  245. data/php/php4/xapian_wrap.cc +27656 -0
  246. data/php/php5/php_xapian.h +319 -0
  247. data/php/php5/xapian.php +1566 -0
  248. data/php/php5/xapian_wrap.cc +24330 -0
  249. data/php/smoketest.php +246 -0
  250. data/php/smoketest4.php +84 -0
  251. data/php/smoketest5.php +79 -0
  252. data/php/util.i +187 -0
  253. data/python/.deps/xapian_wrap.Plo +1 -0
  254. data/python/Makefile +891 -0
  255. data/python/Makefile.am +105 -0
  256. data/python/Makefile.in +891 -0
  257. data/python/doccomments.i +5134 -0
  258. data/python/docs/Makefile +448 -0
  259. data/python/docs/Makefile.am +14 -0
  260. data/python/docs/Makefile.in +448 -0
  261. data/python/docs/examples/simpleexpand.py +98 -0
  262. data/python/docs/examples/simpleindex.py +65 -0
  263. data/python/docs/examples/simplematchdecider.py +78 -0
  264. data/python/docs/examples/simplesearch.py +65 -0
  265. data/python/docs/index.html +420 -0
  266. data/python/except.i +290 -0
  267. data/python/extra.i +1048 -0
  268. data/python/extracomments.i +28 -0
  269. data/python/generate-python-exceptions +189 -0
  270. data/python/generate-python-exceptions.in +189 -0
  271. data/python/modern/xapian.py +5662 -0
  272. data/python/modern/xapian_wrap.cc +35170 -0
  273. data/python/modern/xapian_wrap.h +244 -0
  274. data/python/pythontest.py +1110 -0
  275. data/python/smoketest.py +328 -0
  276. data/python/testsuite.py +382 -0
  277. data/python/util.i +517 -0
  278. data/ruby/.deps/xapian_wrap.Plo +494 -0
  279. data/ruby/.libs/_xapian.bundle +0 -0
  280. data/ruby/.libs/_xapian.bundle.dSYM/Contents/Info.plist +25 -0
  281. data/ruby/.libs/_xapian.bundle.dSYM/Contents/Resources/DWARF/_xapian.bundle +0 -0
  282. data/ruby/.libs/_xapian.la +35 -0
  283. data/ruby/.libs/_xapian.lai +35 -0
  284. data/ruby/Makefile +854 -0
  285. data/ruby/Makefile.am +62 -0
  286. data/ruby/Makefile.in +854 -0
  287. data/ruby/_xapian.la +35 -0
  288. data/ruby/docs/Makefile +487 -0
  289. data/ruby/docs/Makefile.am +50 -0
  290. data/ruby/docs/Makefile.in +487 -0
  291. data/ruby/docs/examples/simpleexpand.rb +98 -0
  292. data/ruby/docs/examples/simpleindex.rb +60 -0
  293. data/ruby/docs/examples/simplematchdecider.rb +74 -0
  294. data/ruby/docs/examples/simplesearch.rb +63 -0
  295. data/ruby/docs/index.html +197 -0
  296. data/ruby/smoketest.rb +211 -0
  297. data/ruby/util.i +232 -0
  298. data/ruby/xapian.rb +280 -0
  299. data/ruby/xapian_wrap.cc +25837 -0
  300. data/ruby/xapian_wrap.h +65 -0
  301. data/ruby/xapian_wrap.lo +12 -0
  302. data/skiptest +2 -0
  303. data/stamp-h1 +1 -0
  304. data/tcl8/.deps/xapian_wrap.Plo +1 -0
  305. data/tcl8/Makefile +835 -0
  306. data/tcl8/Makefile.am +49 -0
  307. data/tcl8/Makefile.in +835 -0
  308. data/tcl8/docs/Makefile +448 -0
  309. data/tcl8/docs/Makefile.am +14 -0
  310. data/tcl8/docs/Makefile.in +448 -0
  311. data/tcl8/docs/examples/simpleexpand.tcl +104 -0
  312. data/tcl8/docs/examples/simpleindex.tcl +68 -0
  313. data/tcl8/docs/examples/simplesearch.tcl +66 -0
  314. data/tcl8/docs/index.html +208 -0
  315. data/tcl8/except.i +48 -0
  316. data/tcl8/pkgIndex.tcl +1 -0
  317. data/tcl8/pkgIndex.tcl.in +1 -0
  318. data/tcl8/run-tcl-test +15 -0
  319. data/tcl8/runtest.tcl +29 -0
  320. data/tcl8/smoketest.tcl +155 -0
  321. data/tcl8/util.i +76 -0
  322. data/tcl8/xapian_wrap.cc +20900 -0
  323. data/xapian-bindings.spec +206 -0
  324. data/xapian-bindings.spec.in +206 -0
  325. data/xapian-version.h +1 -0
  326. data/xapian-version.h.in +1 -0
  327. data/xapian.i +939 -0
  328. metadata +395 -0
@@ -0,0 +1,65 @@
1
+ #!/usr/bin/env python
2
+ #
3
+ # Index each paragraph of a text file as a Xapian document.
4
+ #
5
+ # Copyright (C) 2003 James Aylett
6
+ # Copyright (C) 2004,2007 Olly Betts
7
+ #
8
+ # This program is free software; you can redistribute it and/or
9
+ # modify it under the terms of the GNU General Public License as
10
+ # published by the Free Software Foundation; either version 2 of the
11
+ # License, or (at your option) any later version.
12
+ #
13
+ # This program is distributed in the hope that it will be useful,
14
+ # but WITHOUT ANY WARRANTY; without even the implied warranty of
15
+ # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16
+ # GNU General Public License for more details.
17
+ #
18
+ # You should have received a copy of the GNU General Public License
19
+ # along with this program; if not, write to the Free Software
20
+ # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301
21
+ # USA
22
+
23
+ import sys
24
+ import xapian
25
+ import string
26
+
27
+
28
+ if len(sys.argv) != 2:
29
+ print >> sys.stderr, "Usage: %s PATH_TO_DATABASE" % sys.argv[0]
30
+ sys.exit(1)
31
+
32
+ try:
33
+ # Open the database for update, creating a new database if necessary.
34
+ database = xapian.WritableDatabase(sys.argv[1], xapian.DB_CREATE_OR_OPEN)
35
+
36
+ indexer = xapian.TermGenerator()
37
+ stemmer = xapian.Stem("english")
38
+ indexer.set_stemmer(stemmer)
39
+
40
+ para = ''
41
+ try:
42
+ for line in sys.stdin:
43
+ line = string.strip(line)
44
+ if line == '':
45
+ if para != '':
46
+ # We've reached the end of a paragraph, so index it.
47
+ doc = xapian.Document()
48
+ doc.set_data(para)
49
+
50
+ indexer.set_document(doc)
51
+ indexer.index_text(para)
52
+
53
+ # Add the document to the database.
54
+ database.add_document(doc)
55
+ para = ''
56
+ else:
57
+ if para != '':
58
+ para += ' '
59
+ para += line
60
+ except StopIteration:
61
+ pass
62
+
63
+ except Exception, e:
64
+ print >> sys.stderr, "Exception: %s" % str(e)
65
+ sys.exit(1)
@@ -0,0 +1,78 @@
1
+ #!/usr/bin/env python
2
+ #
3
+ # Simple command-line match decider example
4
+ #
5
+ # Copyright (C) 2003 James Aylett
6
+ # Copyright (C) 2004,2007,2009 Olly Betts
7
+ #
8
+ # This program is free software; you can redistribute it and/or
9
+ # modify it under the terms of the GNU General Public License as
10
+ # published by the Free Software Foundation; either version 2 of the
11
+ # License, or (at your option) any later version.
12
+ #
13
+ # This program is distributed in the hope that it will be useful,
14
+ # but WITHOUT ANY WARRANTY; without even the implied warranty of
15
+ # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16
+ # GNU General Public License for more details.
17
+ #
18
+ # You should have received a copy of the GNU General Public License
19
+ # along with this program; if not, write to the Free Software
20
+ # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301
21
+ # USA
22
+
23
+ import sys
24
+ import xapian
25
+
26
+ # This example runs a query like simplesearch does, but uses a MatchDecider
27
+ # (mymatchdecider) to discard any document for which value 0 is equal to
28
+ # the string passed as the second command line argument.
29
+
30
+ if len(sys.argv) < 4:
31
+ print >> sys.stderr, "Usage: %s PATH_TO_DATABASE AVOID_VALUE QUERY" % sys.argv[0]
32
+ sys.exit(1)
33
+
34
+ class mymatchdecider(xapian.MatchDecider):
35
+ def __init__(self, avoidvalue):
36
+ xapian.MatchDecider.__init__(self)
37
+ self.avoidvalue = avoidvalue
38
+
39
+ def __call__(self, doc):
40
+ return doc.get_value(0) != self.avoidvalue
41
+
42
+ try:
43
+ # Open the database for searching.
44
+ database = xapian.Database(sys.argv[1])
45
+
46
+ # Start an enquire session.
47
+ enquire = xapian.Enquire(database)
48
+
49
+ # Combine the rest of the command line arguments with spaces between
50
+ # them, so that simple queries don't have to be quoted at the shell
51
+ # level.
52
+ avoid_value = sys.argv[2]
53
+ query_string = str.join(' ', sys.argv[3:])
54
+
55
+ # Parse the query string to produce a Xapian::Query object.
56
+ qp = xapian.QueryParser()
57
+ stemmer = xapian.Stem("english")
58
+ qp.set_stemmer(stemmer)
59
+ qp.set_database(database)
60
+ qp.set_stemming_strategy(xapian.QueryParser.STEM_SOME)
61
+ query = qp.parse_query(query_string)
62
+ print "Parsed query is: %s" % str(query)
63
+
64
+ # Find the top 10 results for the query.
65
+ enquire.set_query(query)
66
+ mdecider = mymatchdecider(avoid_value)
67
+ matches = enquire.get_mset(0, 10, None, mdecider)
68
+
69
+ # Display the results.
70
+ print "%i results found." % matches.get_matches_estimated()
71
+ print "Results 1-%i:" % matches.size()
72
+
73
+ for m in matches:
74
+ print "%i: %i%% docid=%i [%s]" % (m.rank + 1, m.percent, m.docid, m.document.get_data())
75
+
76
+ except Exception, e:
77
+ print >> sys.stderr, "Exception: %s" % str(e)
78
+ sys.exit(1)
@@ -0,0 +1,65 @@
1
+ #!/usr/bin/env python
2
+ #
3
+ # Simple command-line search script.
4
+ #
5
+ # Copyright (C) 2003 James Aylett
6
+ # Copyright (C) 2004,2007,2009 Olly Betts
7
+ #
8
+ # This program is free software; you can redistribute it and/or
9
+ # modify it under the terms of the GNU General Public License as
10
+ # published by the Free Software Foundation; either version 2 of the
11
+ # License, or (at your option) any later version.
12
+ #
13
+ # This program is distributed in the hope that it will be useful,
14
+ # but WITHOUT ANY WARRANTY; without even the implied warranty of
15
+ # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16
+ # GNU General Public License for more details.
17
+ #
18
+ # You should have received a copy of the GNU General Public License
19
+ # along with this program; if not, write to the Free Software
20
+ # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301
21
+ # USA
22
+
23
+ import sys
24
+ import xapian
25
+
26
+ # We require at least two command line arguments.
27
+ if len(sys.argv) < 3:
28
+ print >> sys.stderr, "Usage: %s PATH_TO_DATABASE QUERY" % sys.argv[0]
29
+ sys.exit(1)
30
+
31
+ try:
32
+ # Open the database for searching.
33
+ database = xapian.Database(sys.argv[1])
34
+
35
+ # Start an enquire session.
36
+ enquire = xapian.Enquire(database)
37
+
38
+ # Combine the rest of the command line arguments with spaces between
39
+ # them, so that simple queries don't have to be quoted at the shell
40
+ # level.
41
+ query_string = str.join(' ', sys.argv[2:])
42
+
43
+ # Parse the query string to produce a Xapian::Query object.
44
+ qp = xapian.QueryParser()
45
+ stemmer = xapian.Stem("english")
46
+ qp.set_stemmer(stemmer)
47
+ qp.set_database(database)
48
+ qp.set_stemming_strategy(xapian.QueryParser.STEM_SOME)
49
+ query = qp.parse_query(query_string)
50
+ print "Parsed query is: %s" % str(query)
51
+
52
+ # Find the top 10 results for the query.
53
+ enquire.set_query(query)
54
+ matches = enquire.get_mset(0, 10)
55
+
56
+ # Display the results.
57
+ print "%i results found." % matches.get_matches_estimated()
58
+ print "Results 1-%i:" % matches.size()
59
+
60
+ for m in matches:
61
+ print "%i: %i%% docid=%i [%s]" % (m.rank + 1, m.percent, m.docid, m.document.get_data())
62
+
63
+ except Exception, e:
64
+ print >> sys.stderr, "Exception: %s" % str(e)
65
+ sys.exit(1)
@@ -0,0 +1,420 @@
1
+ <html><head><title>Python bindings for Xapian</title></head>
2
+ <body>
3
+ <h1>Python bindings for Xapian</h1>
4
+
5
+ <p>
6
+ The Python bindings for Xapian are packaged in the <code>xapian</code> module,
7
+ and largely follow the C++ API, with the following differences and
8
+ additions. Python strings and lists, etc., are converted automatically
9
+ in the bindings, so generally it should just work as expected.
10
+ </p>
11
+
12
+ <p>
13
+ The <code>examples</code> subdirectory contains examples showing how to use the
14
+ Python bindings based on the simple examples from <code>xapian-examples</code>:
15
+ <a href="examples/simpleindex.py">simpleindex.py</a>,
16
+ <a href="examples/simplesearch.py">simplesearch.py</a>,
17
+ <a href="examples/simpleexpand.py">simpleexpand.py</a>.
18
+ There's also
19
+ <a href="examples/simplematchdecider.py">simplematchdecider.py</a>
20
+ which shows how to define a MatchDecider in Python.
21
+ </p>
22
+
23
+ <p>
24
+ The Python bindings come with a test suite, consisting of two test files:
25
+ <code>smoketest.py</code> and <code>pythontest.py</code>. These are run by the
26
+ "<code>make check</code>" command, or may be run manually. By default, they
27
+ will display the names of any tests which failed, and then display a count of
28
+ tests which run and which failed. The verbosity may be increased by setting
29
+ the "<code>VERBOSE</code>" environment variable: a value of 1 will display
30
+ detailed information about failures, and a value of 2 will display further
31
+ information about the progress of tests.
32
+ </p>
33
+
34
+ <h2>Exceptions</h2>
35
+
36
+ <p>
37
+ Xapian exceptions are translated into Python exceptions with the same names
38
+ and inheritance hierarchy as the C++ exception classes. The base class of
39
+ all Xapian exceptions is the <code>xapian.Error</code> class, and this in
40
+ turn is a child of the standard python <code>exceptions.Exception</code>
41
+ class.
42
+ </p>
43
+ <p>
44
+ This means that programs can trap all xapian exceptions using "<code>except
45
+ xapian.Error</code>", and can trap all exceptions which don't indicate that
46
+ the program should terminate using "<code>except Exception</code>".
47
+ </p>
48
+
49
+ <h2>Unicode</h2>
50
+
51
+ <p>
52
+ The xapian Python bindings accept unicode strings as well as simple strings
53
+ (ie, "str" type strings) at all places in the API which accept string data.
54
+ Any unicode strings supplied will automatically be translated into UTF-8
55
+ simple strings before being passed to the Xapian core. The Xapian core is
56
+ largely agnostic about character encoding, but in those places where it does
57
+ process data in a character encoding dependent way it assumes that the data
58
+ is in UTF-8. The Xapian Python bindings always return string data as simple
59
+ strings.
60
+ </p>
61
+ <p>
62
+ Therefore, in order to avoid issues with character encodings, you should
63
+ always pass text data to Xapian as unicode strings, or UTF-8 encoded simple
64
+ strings. There is, however, no requirement for simple strings passed into
65
+ Xapian to be valid UTF-8 encoded strings, unless they are being passed to a
66
+ text processing routine (such as the query parser, or the stemming
67
+ algorithms). For example, it is perfectly valid to pass arbitrary binary
68
+ data in a simple string to the <code>xapian.Document.set_data()</code>
69
+ method.
70
+ </p>
71
+ <p>
72
+ It is often useful to normalise unicode data before passing it to Xapian -
73
+ Xapian currently has no built-in support for normalising unicode
74
+ representations of data. The standard python module
75
+ "<code>unicodedata</code>" provides support for normalising unicode: you
76
+ probably want the "<code>NFKC</code>" normalisation scheme: in other words,
77
+ use something like
78
+ </p>
79
+ <pre>
80
+ unicodedata.normalize('NFKC', u'foo')
81
+ </pre>
82
+ <p>
83
+ to normalise the string "foo" before passing it to Xapian.
84
+ </p>
85
+
86
+ <h2>Iterators</h2>
87
+
88
+ <p>
89
+ The iterator classes in the Xapian C++ API are wrapped in a "Pythonic" style.
90
+ The following are supported (where marked as default iterator, it means
91
+ <code>__iter__()</code> does the right
92
+ thing so you can for instance use <code>for term in document</code> to
93
+ iterate over terms in a Document object):
94
+ </p>
95
+
96
+ <table title="Python iterators">
97
+ <thead><td>Class</td><td>Method</td><td>Equivalent to</td><td>Iterator type</td></thead>
98
+ <tr><td><code>MSet</code></td><td>default iterator</td><td><code>begin()</code></td><td><code>MSetIter</code></td></tr>
99
+ <tr><td><code>ESet</code></td><td>default iterator</td><td><code>begin()</code></td><td><code>ESetIter</code></td></tr>
100
+ <tr><td><code>Enquire</code></td><td><code>matching_terms()</code></td><td><code>get_matching_terms_begin()</code></td><td><code>TermIter</code></td></tr>
101
+ <tr><td><code>Query</code></td><td>default iterator</td><td><code>get_terms_begin()</code></td><td><code>TermIter</code></td></tr>
102
+ <tr><td><code>Database</code></td><td><code>allterms()</code> (also as default iterator)</td><td><code>allterms_begin()</code></td><td><code>TermIter</code></td></tr>
103
+ <tr><td><code>Database</code></td><td><code>postlist(tname)</code></td><td><code>postlist_begin(tname)</code></td><td><code>PostingIter</code></td></tr>
104
+ <tr><td><code>Database</code></td><td><code>termlist(docid)</code></td><td><code>termlist_begin(docid)</code></td><td><code>TermIter</code></td></tr>
105
+ <tr><td><code>Database</code></td><td><code>positionlist(docid, tname)</code></td><td><code>positionlist_begin(docid, tname)</code></td><td><code>PositionIter</code></td></tr>
106
+ <tr><td><code>Database</code></td><td><code>metadata_keys(prefix)</code></td><td><code>metadata_keys_begin(prefix)</code></td><td><code>TermIter</code></td></tr>
107
+ <tr><td><code>Database</code></td><td><code>spellings()</code></td><td><code>spellings_begin(term)</code></td><td><code>TermIter</code></td></tr>
108
+ <tr><td><code>Database</code></td><td><code>synonyms(term)</code></td><td><code>synonyms_begin(term)</code></td><td><code>TermIter</code></td></tr>
109
+ <tr><td><code>Database</code></td><td><code>synonym_keys(prefix)</code></td><td><code>synonym_keys_begin(prefix)</code></td><td><code>TermIter</code></td></tr>
110
+ <tr><td><code>Document</code></td><td><code>values()</code></td><td><code>values_begin()</code></td><td><code>ValueIter</code></td></tr>
111
+ <tr><td><code>Document</code></td><td><code>termlist()</code> (also as default iterator)</td><td><code>termlist_begin()</code></td><td><code>TermIter</code></td></tr>
112
+ <tr><td><code>QueryParser</code></td><td><code>stoplist()</code></td><td><code>stoplist_begin()</code></td><td><code>TermIter</code></td></tr>
113
+ <tr><td><code>QueryParser</code></td><td><code>unstemlist(tname)</code></td><td><code>unstem_begin(tname)</code></td><td><code>TermIter</code></td></tr>
114
+ </table>
115
+
116
+ <p>
117
+ The pythonic iterators return Python objects, with properties available as
118
+ available as attribute values, with lazy evaluation where appropriate. An
119
+ exception is the <code>PositionIter</code> object returned by
120
+ <code>Database.positionlist</code>, which returns an integer.
121
+ </p>
122
+
123
+ <p>
124
+ The lazy evaluation is mainly transparent, but does become visible in one situation: if you keep an object returned by an iterator, without evaluating its properties to force the lazy evaluation to happen, and then move the iterator forward, the object may no longer be able to efficiently perform the lazy evaluation. In this situation, an exception will be raised indicating that the information requested wasn't available. This will only happen for a few of the properties - most are either not evaluated lazily (because the underlying Xapian implementation doesn't evaluate them lazily, so there's no advantage in lazy evaluation), or can be accessed even after the iterator has moved. The simplest work around is simply to evaluate any properties you wish to use which are affected by this before moving the iterator. The complete set of iterator properties affected by this is:
125
+ </p>
126
+
127
+ <ul>
128
+ <li>
129
+ Database.allterms (also accessible as Database.__iter__): <b>termfreq</b>
130
+ </li><li>
131
+ Database.termlist: <b>termfreq</b> and <b>positer</b>
132
+ </li><li>
133
+ Document.termlist (also accessible as Document.__iter__): <b>termfreq</b> and <b>positer</b>
134
+ </li><li>
135
+ Database.postlist: <b>positer</b>
136
+ </li>
137
+ </ul>
138
+
139
+ <p>
140
+ In older releases, the pythonic iterators returned lists representing the
141
+ appropriate item when their <code>next()</code> method was called, except
142
+ PositionIter which just returned a single value. For backward compatibility,
143
+ this sequence API is still supported so existing code should continue to work,
144
+ but its use is now deprecated and it will be removed in Xapian 1.1.0, so we
145
+ recommend users migrate to the new API.
146
+ </p>
147
+
148
+ <table title="Sequence API equivalents">
149
+ <thead><td>Class</td><td>Returns</td></thead>
150
+ <tr><td><code>MSetIter</code></td><td>[docid, weight, rank, percentage, document]</td></tr>
151
+ <tr><td><code>ESetIter</code></td><td>[term, weight]</td></tr>
152
+ <tr><td><code>TermIter</code></td><td>[term, wdf, termfreq, position iterator]</td></tr>
153
+ <tr><td><code>PostingIter</code></td><td>[docid, doclength, wdf, position iterator]</td></tr>
154
+ <tr><td><code>PositionIter</code></td><td>termpos</td></tr>
155
+ <tr><td><code>ValueIter</code></td><td>[valueno, value]</td></tr>
156
+ </table>
157
+
158
+ <h2>Non-Pythonic Iterators</h2>
159
+
160
+ <p>
161
+ Before the pythonic iterator wrappers were added, the python bindings provided
162
+ thin wrappers around the C++ iterators. However, these iterators don't behave
163
+ like most iterators do in Python, so the pythonic iterators were implemented to
164
+ replace them. The non-pythonic iterators are still available to allow existing
165
+ code to continue to work, but they're now deprecated and we plan to remove them
166
+ in Xapian 1.3.0. The documentation below is provided to aid migration away from
167
+ them.
168
+ </p>
169
+
170
+ <p>
171
+ All non-pythonic iterators support <code>next()</code> and
172
+ <code>equals()</code> methods
173
+ to move through and test iterators (as for all language bindings).
174
+ MSetIterator and ESetIterator also support <code>prev()</code>.
175
+ Python-wrapped iterators also support direct comparison, so something like:
176
+ </p>
177
+
178
+ <pre>
179
+ m=mset.begin()
180
+ while m!=mset.end():
181
+ # do something
182
+ m.next()
183
+ </pre>
184
+
185
+ <p>
186
+ C++ iterators are often dereferenced to get information, eg
187
+ <code>(*it)</code>. With Python these are all mapped to named methods, as
188
+ follows:
189
+ </p>
190
+
191
+ <table title="Iterator deferencing methods">
192
+ <thead><td>Iterator</td><td>Dereferencing method</td></thead>
193
+ <tr><td>PositionIterator</td> <td><code>get_termpos()</code></td></tr>
194
+ <tr><td>PostingIterator</td> <td><code>get_docid()</code></td></tr>
195
+ <tr><td>TermIterator</td> <td><code>get_term()</code></td></tr>
196
+ <tr><td>ValueIterator</td> <td><code>get_value()</code></td></tr>
197
+ <tr><td>MSetIterator</td> <td><code>get_docid()</code></td></tr>
198
+ <tr><td>ESetIterator</td> <td><code>get_term()</code></td></tr>
199
+ </table>
200
+
201
+ <p>
202
+ Other methods, such as <code>MSetIterator.get_document()</code>, are
203
+ available unchanged.
204
+ </p>
205
+
206
+ <h2>MSet</h2>
207
+
208
+ <p>
209
+ MSet objects have some additional methods to simplify access (these
210
+ work using the C++ array dereferencing):
211
+ </p>
212
+
213
+ <table title="MSet additional methods">
214
+ <thead><td>Method name</td><td>Explanation</td></thead>
215
+ <tr><td><code>get_hit(index)</code></td><td>returns MSetItem at index</td></tr>
216
+ <tr><td><code>get_document_percentage(index)</code></td><td><code>convert_to_percent(get_hit(index))</code></td></tr>
217
+ <tr><td><code>get_document(index)</code></td><td><code>get_hit(index).get_document()</code></td></tr>
218
+ <tr><td><code>get_docid(index)</code></td><td><code>get_hit(index).get_docid()</code></td></tr>
219
+ </table>
220
+
221
+ <p>
222
+ Additionally, the MSet has a property, <code>mset.items</code>, which returns a
223
+ list of tuples representing the MSet; this may be more convenient than using
224
+ an MSetIter. The members of the tuple are as follows.
225
+ </p>
226
+
227
+ <table title="MSet.items member members">
228
+ <thead><td>Index</td><td>Contents</td></thead>
229
+ <tr><td><code>xapian.MSET_DID</code></td><td>Document id</td></tr>
230
+ <tr><td><code>xapian.MSET_WT</code></td><td>Weight</td></tr>
231
+ <tr><td><code>xapian.MSET_RANK</code></td><td>Rank</td></tr>
232
+ <tr><td><code>xapian.MSET_PERCENT</code></td><td>Percentage weight</td></tr>
233
+ <tr><td><code>xapian.MSET_DOCUMENT</code></td><td>Document object</td></tr>
234
+ </table>
235
+
236
+ <p>
237
+ Two MSet objects are equal if they have the same number and maximum possible
238
+ number of members, and if every document member of the first MSet exists at the
239
+ same index in the second MSet, with the same weight.
240
+ </p>
241
+
242
+ <h2>ESet</h2>
243
+
244
+ <p>
245
+ The ESet has a property, <code>eset.items</code>, which returns a list of
246
+ tuples representing the ESet; this may be more convenient than using the
247
+ ESetIterator. The members of the tuple are as follows.
248
+ </p>
249
+
250
+ <table title="ESet.items member members">
251
+ <thead><td>Index</td><td>Contents</td></thead>
252
+ <tr><td><code>xapian.ESET_TNAME</code></td><td>Term name</td></tr>
253
+ <tr><td><code>xapian.ESET_WT</code></td><td>Weight</td></tr>
254
+ </table>
255
+
256
+ <h2>Non-Class Functions</h2>
257
+
258
+ <p>The C++ API contains a few non-class functions (the Database factory
259
+ functions, and some functions reporting version information), which are
260
+ wrapped like so for Python:
261
+ <ul>
262
+ <ul>
263
+ <li> <code>Xapian::version_string()</code> is wrapped as <code>xapian.version_string()</code>
264
+ <li> <code>Xapian::major_version()</code> is wrapped as <code>xapian.major_version()</code>
265
+ <li> <code>Xapian::minor_version()</code> is wrapped as <code>xapian.minor_version()</code>
266
+ <li> <code>Xapian::revision()</code> is wrapped as <code>xapian.revision()</code>
267
+ </ul>
268
+ <ul>
269
+ <li> <code>Xapian::Auto::open_stub()</code> is wrapped as <code>xapian.open_stub()</code>
270
+ <li> <code>Xapian::Flint::open()</code> is wrapped as <code>xapian.flint_open()</code>
271
+ <li> <code>Xapian::InMemory::open()</code> is wrapped as <code>xapian.inmemory_open()</code>
272
+ <li> <code>Xapian::Quartz::open()</code> is wrapped as <code>xapian.quartz_open()</code>
273
+ <li> <code>Xapian::Remote::open()</code> is wrapped as <code>xapian.remote_open()</code> (both
274
+ the TCP and "program" versions are wrapped - the SWIG wrapper checks the parameter list to
275
+ decide which to call).
276
+ <li> <code>Xapian::Remote::open_writable()</code> is wrapped as <code>xapian.remote_open_writable()</code> (both
277
+ the TCP and "program" versions are wrapped - the SWIG wrapper checks the parameter list to
278
+ decide which to call).
279
+ </ul>
280
+ </ul>
281
+
282
+ <h2>Query</h2>
283
+
284
+ <p>
285
+ In C++ there's a Xapian::Query constructor which takes a query operator and
286
+ start/end iterators specifying a number of terms or queries, plus an optional
287
+ parameter. In Python, this is wrapped to accept any Python sequence (for
288
+ example a list or tuple) to give the terms/queries, and you can specify
289
+ a mixture of terms and queries if you wish. For example:
290
+ </p>
291
+
292
+ <pre>
293
+ subq = xapian.Query(xapian.Query.OP_AND, "hello", "world")
294
+ q = xapian.Query(xapian.Query.OP_AND, [subq, "foo", xapian.Query("bar", 2)])
295
+ </pre>
296
+
297
+ <h2>Enquire</h2>
298
+
299
+ <p>
300
+ There is an additional method <code>get_matching_terms()</code> which takes
301
+ an MSetIterator and returns a list of terms in the current query which
302
+ match the document given by that iterator. You may find this
303
+ more convenient than using the TermIterator directly.
304
+ </p>
305
+
306
+ <h2>MatchDecider</h2>
307
+
308
+ <p>
309
+ Custom MatchDeciders can be created in Python; simply subclass
310
+ xapian.MatchDecider, ensure you call the super-constructor, and define a
311
+ __call__ method that will do the work. The simplest example (which does nothing
312
+ useful) would be as follows:
313
+ </p>
314
+
315
+ <pre>
316
+ class mymatchdecider(xapian.MatchDecider):
317
+ def __init__(self):
318
+ xapian.MatchDecider.__init__(self)
319
+
320
+ def __call__(self, doc):
321
+ return 1
322
+ </pre>
323
+
324
+ <h2>ValueRangeProcessors</h2>
325
+
326
+ <p>
327
+ The ValueRangeProcessor class (and its subclasses) provide an operator() method
328
+ (which is exposed in python as a __call__() method, making the class instances
329
+ into callables). This method checks whether a beginning and end of a range are
330
+ in a format understood by the ValueRangeProcessor, and if so, converts the
331
+ beginning and end into strings which sort appropriately. ValueRangeProcessors
332
+ can be defined in python (and then passed to the QueryParser), or there are
333
+ several default built-in ones which can be used.
334
+ </p>
335
+
336
+ <p>
337
+ Unfortunately, in C++ the operator() method takes two std::string arguments by
338
+ reference, and returns values by modifying these arguments. This is not
339
+ possible in Python, since strings are immutable objects. Instead, in the
340
+ Python implementation, when the __call__ method is called, the resulting values
341
+ of these arguments are returned as part of a tuple. The operator() method in
342
+ C++ returns a value number; the return value of __call__ in python consists of
343
+ a 3-tuple starting with this value number, followed by the returned "begin"
344
+ value, followed by the returned "end" value. For example:
345
+ </p>
346
+
347
+ <pre>
348
+ vrp = xapian.NumberValueRangeProcessor(0, '$', True)
349
+ a = '$10'
350
+ b = '20'
351
+ slot, a, b = vrp(a, b)
352
+ </pre>
353
+
354
+ <p>
355
+ Additionally, a ValueRangeProcessor may be implemented in Python. The Python
356
+ implementation should override the __call__() method with its own
357
+ implementation, and, again, since it cannot return values by reference, it
358
+ should return a tuple of (value number, begin, end). For example:
359
+ </p>
360
+
361
+ <pre>
362
+ class MyVRP(xapian.ValueRangeProcessor):
363
+ def __init__(self):
364
+ xapian.ValueRangeProcessor.__init__(self)
365
+ def __call__(self, begin, end):
366
+ return (7, "A"+begin, "B"+end)
367
+ </pre>
368
+
369
+
370
+ <h2>Apache and mod_python/mod_wsgi</h2>
371
+
372
+ <p>
373
+ By default, both mod_python and mod_wsgi use a separate sub-interpreter for
374
+ each application. However, Python's sub-interpreter support is incompatible
375
+ with the simplified GIL state API which SWIG-generated Python bindings use
376
+ by default. So to avoid deadlocks, you need to tell mod_python and mod_wsgi
377
+ to run applications which use Xapian in the main interpreter as detailed below.
378
+ </p>
379
+
380
+ <p>
381
+ This restriction could be removed - the details are in Xapian's bugtracker - see
382
+ <a href="http://trac.xapian.org/ticket/364">ticket #364</a> for details of
383
+ what needs doing.
384
+ </p>
385
+
386
+ <h3 id="mod_python">mod_python</h3>
387
+
388
+ <p>
389
+ You need to set this option in the Apache configuration section for all
390
+ mod_python scripts which use Xapian:
391
+ </p>
392
+ <pre>PythonInterpreter main_interpreter
393
+ </pre>
394
+
395
+ <p>
396
+ You may also need to use Python &gt;= 2.4 (due to <a
397
+ href="http://issues.apache.org/jira/browse/MODPYTHON-217">problems in Python
398
+ 2.3 with the APIs the code uses</a>).
399
+ </p>
400
+
401
+ <h3 id="mod_wsgi">mod_wsgi</h3>
402
+
403
+ <p>
404
+ You need to set the
405
+ <a href="http://code.google.com/p/modwsgi/wiki/ConfigurationDirectives#WSGIApplicationGroup">WSGIApplicationGroup option</a> like so:
406
+ </p>
407
+
408
+ <pre>WSGIApplicationGroup %{GLOBAL}
409
+ </pre>
410
+
411
+ <p>
412
+ The mod_wsgi documentation
413
+ <a href="http://code.google.com/p/modwsgi/wiki/ApplicationIssues#Python_Simplified_GIL_State_API">also discusses this issue</a>.
414
+ </p>
415
+
416
+ <address>
417
+ Last updated $Date: 2009-05-23 08:20:54 +0100 (Sat, 23 May 2009) $
418
+ </address>
419
+ </body>
420
+ </html>