iudex-core 1.0.0-java → 1.1.0-java

data/config/mojibake

@@ -0,0 +1,268 @@
+# -*- coding: utf-8 -*- mojibake: 1.0.0
+/Â[\u0080\u0081\u0082\u0083\u0084\u0085\u0086\u0087\u0088\u0089\u008A\u008B\u008C\u008D\u008E\u008F\u0090\u0091\u0092\u0093\u0094\u0095\u0096\u0097\u0098\u0099\u009A\u009B\u009C\u009D\u009E\u009F\u00A0¡¢£¤¥¦§¨©ª«¬­®¯°±²³´µ¶·¸¹º»¼½¾¿ŒœŠšŸŽžƒˆ˜–—‘’‚“”„†‡•…‰‹›€™\uFFFD]|Ã[\u0080\u0081\u0082\u0083\u0084\u0085\u0086\u0087\u0088\u0089\u008A\u008B\u008C\u008D\u008E\u008F\u0090\u0091\u0092\u0093\u0094\u0095\u0096\u0097\u0098\u0099\u009A\u009B\u009C\u009D\u009E\u009F\u00A0¡¢£¤¥¦§¨©ª«¬­®¯°±²³´µ¶·¸¹º»¼½¾¿ŒœŠšŸŽžƒˆ˜–—‘’‚“”„†‡•…‰‹›€™\uFFFD]|Å[\u0092\u0093\u00A0¡¸½¾’“]|Æ[\u0092’]|Ë[\u0086\u009Cœ†]|â(\u0080[\u0080\u0081\u0082\u0083\u0084\u0085\u0086\u0087\u0088\u0089\u008A\u008B\u0093\u0094\u0098\u0099\u009A\u009C\u009D\u009E\u00A0¡¢¦°¹º]|\u0081\u00A0|\u0082¬|\u0084¢|‚¬|„¢|€[\u0081\u009D\u00A0¡¢¦°¹ºœŠšžƒˆ˜‚“”„†‡…‰‹€™\uFFFD]|\uFFFD\u00A0)|ï(»¿|¿[½¾])/
+
+Moji	UNICODE  	Org	CODE
++----	---- ---- ----	-----	---+
+[€]	00C2 0080	[€]	0080
+[]	00C2 0081	[]	0081
+[‚]	00C2 0082	[‚]	0082
+[ƒ]	00C2 0083	[ƒ]	0083
+[„]	00C2 0084	[„]	0084
+[…]	00C2 0085	[…]	0085
+[†]	00C2 0086	[†]	0086
+[‡]	00C2 0087	[‡]	0087
+[ˆ]	00C2 0088	[ˆ]	0088
+[‰]	00C2 0089	[‰]	0089
+[Š]	00C2 008A	[Š]	008A
+[‹]	00C2 008B	[‹]	008B
+[Œ]	00C2 008C	[Œ]	008C
+[]	00C2 008D	[]	008D
+[Ž]	00C2 008E	[Ž]	008E
+[]	00C2 008F	[]	008F
+[]	00C2 0090	[]	0090
+[‘]	00C2 0091	[‘]	0091
+[’]	00C2 0092	[’]	0092
+[“]	00C2 0093	[“]	0093
+[”]	00C2 0094	[”]	0094
+[•]	00C2 0095	[•]	0095
+[–]	00C2 0096	[–]	0096
+[—]	00C2 0097	[—]	0097
+[˜]	00C2 0098	[˜]	0098
+[™]	00C2 0099	[™]	0099
+[š]	00C2 009A	[š]	009A
+[›]	00C2 009B	[›]	009B
+[œ]	00C2 009C	[œ]	009C
+[]	00C2 009D	[]	009D
+[ž]	00C2 009E	[ž]	009E
+[Ÿ]	00C2 009F	[Ÿ]	009F
+[ ]	00C2 00A0	[ ]	00A0
+[¡]	00C2 00A1	[¡]	00A1
+[¢]	00C2 00A2	[¢]	00A2
+[£]	00C2 00A3	[£]	00A3
+[¤]	00C2 00A4	[¤]	00A4
+[Â¥]	00C2 00A5	[¥]	00A5
+[¦]	00C2 00A6	[¦]	00A6
+[§]	00C2 00A7	[§]	00A7
+[¨]	00C2 00A8	[¨]	00A8
+[©]	00C2 00A9	[©]	00A9
+[ª]	00C2 00AA	[ª]	00AA
+[«]	00C2 00AB	[«]	00AB
+[¬]	00C2 00AC	[¬]	00AC
+[­]	00C2 00AD	[­]	00AD
+[®]	00C2 00AE	[®]	00AE
+[¯]	00C2 00AF	[¯]	00AF
+[°]	00C2 00B0	[°]	00B0
+[±]	00C2 00B1	[±]	00B1
+[²]	00C2 00B2	[²]	00B2
+[³]	00C2 00B3	[³]	00B3
+[´]	00C2 00B4	[´]	00B4
+[µ]	00C2 00B5	[µ]	00B5
+[¶]	00C2 00B6	[¶]	00B6
+[·]	00C2 00B7	[·]	00B7
+[¸]	00C2 00B8	[¸]	00B8
+[¹]	00C2 00B9	[¹]	00B9
+[º]	00C2 00BA	[º]	00BA
+[»]	00C2 00BB	[»]	00BB
+[¼]	00C2 00BC	[¼]	00BC
+[½]	00C2 00BD	[½]	00BD
+[¾]	00C2 00BE	[¾]	00BE
+[¿]	00C2 00BF	[¿]	00BF
+[ÂŒ]	00C2 0152	[Œ]	008C
+[Âœ]	00C2 0153	[œ]	009C
+[Š]	00C2 0160	[Š]	008A
+[š]	00C2 0161	[š]	009A
+[Ÿ]	00C2 0178	[Ÿ]	009F
+[ÂŽ]	00C2 017D	[Ž]	008E
+[ž]	00C2 017E	[ž]	009E
+[ƒ]	00C2 0192	[ƒ]	0083
+[ˆ]	00C2 02C6	[ˆ]	0088
+[˜]	00C2 02DC	[˜]	0098
+[–]	00C2 2013	[–]	0096
+[—]	00C2 2014	[—]	0097
+[‘]	00C2 2018	[‘]	0091
+[Â’]	00C2 2019	[’]	0092
+[‚]	00C2 201A	[‚]	0082
+[“]	00C2 201C	[“]	0093
+[”]	00C2 201D	[”]	0094
+[„]	00C2 201E	[„]	0084
+[†]	00C2 2020	[†]	0086
+[‡]	00C2 2021	[‡]	0087
+[•]	00C2 2022	[•]	0095
+[Â…]	00C2 2026	[…]	0085
+[‰]	00C2 2030	[‰]	0089
+[‹]	00C2 2039	[‹]	008B
+[›]	00C2 203A	[›]	009B
+[€]	00C2 20AC	[€]	0080
+[™]	00C2 2122	[™]	0099
+[Â�]	00C2 FFFD	[]	0081
+[À]	00C3 0080	[À]	00C0
+[Á]	00C3 0081	[Á]	00C1
+[Â]	00C3 0082	[Â]	00C2
+[Ã]	00C3 0083	[Ã]	00C3
+[Ä]	00C3 0084	[Ä]	00C4
+[Å]	00C3 0085	[Å]	00C5
+[Æ]	00C3 0086	[Æ]	00C6
+[Ç]	00C3 0087	[Ç]	00C7
+[È]	00C3 0088	[È]	00C8
+[É]	00C3 0089	[É]	00C9
+[Ê]	00C3 008A	[Ê]	00CA
+[Ë]	00C3 008B	[Ë]	00CB
+[Ì]	00C3 008C	[Ì]	00CC
+[Í]	00C3 008D	[Í]	00CD
+[Î]	00C3 008E	[Î]	00CE
+[Ï]	00C3 008F	[Ï]	00CF
+[Ð]	00C3 0090	[Ð]	00D0
+[Ñ]	00C3 0091	[Ñ]	00D1
+[Ò]	00C3 0092	[Ò]	00D2
+[Ó]	00C3 0093	[Ó]	00D3
+[Ô]	00C3 0094	[Ô]	00D4
+[Õ]	00C3 0095	[Õ]	00D5
+[Ö]	00C3 0096	[Ö]	00D6
+[×]	00C3 0097	[×]	00D7
+[Ø]	00C3 0098	[Ø]	00D8
+[Ù]	00C3 0099	[Ù]	00D9
+[Ú]	00C3 009A	[Ú]	00DA
+[Û]	00C3 009B	[Û]	00DB
+[Ü]	00C3 009C	[Ü]	00DC
+[Ý]	00C3 009D	[Ý]	00DD
+[Þ]	00C3 009E	[Þ]	00DE
+[ß]	00C3 009F	[ß]	00DF
+[à]	00C3 00A0	[à]	00E0
+[á]	00C3 00A1	[á]	00E1
+[â]	00C3 00A2	[â]	00E2
+[ã]	00C3 00A3	[ã]	00E3
+[ä]	00C3 00A4	[ä]	00E4
+[Ã¥]	00C3 00A5	[å]	00E5
+[æ]	00C3 00A6	[æ]	00E6
+[ç]	00C3 00A7	[ç]	00E7
+[è]	00C3 00A8	[è]	00E8
+[é]	00C3 00A9	[é]	00E9
+[ê]	00C3 00AA	[ê]	00EA
+[ë]	00C3 00AB	[ë]	00EB
+[ì]	00C3 00AC	[ì]	00EC
+[í]	00C3 00AD	[í]	00ED
+[î]	00C3 00AE	[î]	00EE
+[ï]	00C3 00AF	[ï]	00EF
+[ð]	00C3 00B0	[ð]	00F0
+[ñ]	00C3 00B1	[ñ]	00F1
+[ò]	00C3 00B2	[ò]	00F2
+[ó]	00C3 00B3	[ó]	00F3
+[ô]	00C3 00B4	[ô]	00F4
+[õ]	00C3 00B5	[õ]	00F5
+[ö]	00C3 00B6	[ö]	00F6
+[÷]	00C3 00B7	[÷]	00F7
+[ø]	00C3 00B8	[ø]	00F8
+[ù]	00C3 00B9	[ù]	00F9
+[ú]	00C3 00BA	[ú]	00FA
+[û]	00C3 00BB	[û]	00FB
+[ü]	00C3 00BC	[ü]	00FC
+[ý]	00C3 00BD	[ý]	00FD
+[þ]	00C3 00BE	[þ]	00FE
+[ÿ]	00C3 00BF	[ÿ]	00FF
+[ÃŒ]	00C3 0152	[Ì]	00CC
+[Ãœ]	00C3 0153	[Ü]	00DC
+[Ê]	00C3 0160	[Ê]	00CA
+[Ú]	00C3 0161	[Ú]	00DA
+[ß]	00C3 0178	[ß]	00DF
+[ÃŽ]	00C3 017D	[Î]	00CE
+[Þ]	00C3 017E	[Þ]	00DE
+[Ã]	00C3 0192	[Ã]	00C3
+[È]	00C3 02C6	[È]	00C8
+[Ø]	00C3 02DC	[Ø]	00D8
+[Ö]	00C3 2013	[Ö]	00D6
+[×]	00C3 2014	[×]	00D7
+[Ñ]	00C3 2018	[Ñ]	00D1
+[Ã’]	00C3 2019	[Ò]	00D2
+[Â]	00C3 201A	[Â]	00C2
+[Ó]	00C3 201C	[Ó]	00D3
+[Ô]	00C3 201D	[Ô]	00D4
+[Ä]	00C3 201E	[Ä]	00C4
+[Æ]	00C3 2020	[Æ]	00C6
+[Ç]	00C3 2021	[Ç]	00C7
+[Õ]	00C3 2022	[Õ]	00D5
+[Ã…]	00C3 2026	[Å]	00C5
+[É]	00C3 2030	[É]	00C9
+[Ë]	00C3 2039	[Ë]	00CB
+[Û]	00C3 203A	[Û]	00DB
+[À]	00C3 20AC	[À]	00C0
+[Ù]	00C3 2122	[Ù]	00D9
+[Ã�]	00C3 FFFD	[Á]	00C1
+[Œ]	00C5 0092	[Œ]	0152
+[œ]	00C5 0093	[œ]	0153
+[Å ]	00C5 00A0	[Š]	0160
+[Å¡]	00C5 00A1	[š]	0161
+[Ÿ]	00C5 00B8	[Ÿ]	0178
+[Ž]	00C5 00BD	[Ž]	017D
+[ž]	00C5 00BE	[ž]	017E
+[Å’]	00C5 2019	[Œ]	0152
+[Å“]	00C5 201C	[œ]	0153
+[ƒ]	00C6 0092	[ƒ]	0192
+[Æ’]	00C6 2019	[ƒ]	0192
+[ˆ]	00CB 0086	[ˆ]	02C6
+[˜]	00CB 009C	[˜]	02DC
+[Ëœ]	00CB 0153	[˜]	02DC
+[ˆ]	00CB 2020	[ˆ]	02C6
+[ ]	00E2 0080 0080	[ ]	2000
+[ ]	00E2 0080 0081	[ ]	2001
+[ ]	00E2 0080 0082	[ ]	2002
+[ ]	00E2 0080 0083	[ ]	2003
+[ ]	00E2 0080 0084	[ ]	2004
+[ ]	00E2 0080 0085	[ ]	2005
+[ ]	00E2 0080 0086	[ ]	2006
+[ ]	00E2 0080 0087	[ ]	2007
+[ ]	00E2 0080 0088	[ ]	2008
+[ ]	00E2 0080 0089	[ ]	2009
+[ ]	00E2 0080 008A	[ ]	200A
+[​]	00E2 0080 008B	[​]	200B
+[–]	00E2 0080 0093	[–]	2013
+[—]	00E2 0080 0094	[—]	2014
+[‘]	00E2 0080 0098	[‘]	2018
+[’]	00E2 0080 0099	[’]	2019
+[‚]	00E2 0080 009A	[‚]	201A
+[“]	00E2 0080 009C	[“]	201C
+[”]	00E2 0080 009D	[”]	201D
+[„]	00E2 0080 009E	[„]	201E
+[†]	00E2 0080 00A0	[†]	2020
+[‡]	00E2 0080 00A1	[‡]	2021
+[•]	00E2 0080 00A2	[•]	2022
+[…]	00E2 0080 00A6	[…]	2026
+[‰]	00E2 0080 00B0	[‰]	2030
+[‹]	00E2 0080 00B9	[‹]	2039
+[›]	00E2 0080 00BA	[›]	203A
+[⁠]	00E2 0081 00A0	[⁠]	2060
+[€]	00E2 0082 00AC	[€]	20AC
+[™]	00E2 0084 00A2	[™]	2122
+[€]	00E2 201A 00AC	[€]	20AC
+[â„¢]	00E2 201E 00A2	[™]	2122
+[ ]	00E2 20AC 0081	[ ]	2001
+[”]	00E2 20AC 009D	[”]	201D
+[†]	00E2 20AC 00A0	[†]	2020
+[‡]	00E2 20AC 00A1	[‡]	2021
+[•]	00E2 20AC 00A2	[•]	2022
+[…]	00E2 20AC 00A6	[…]	2026
+[‰]	00E2 20AC 00B0	[‰]	2030
+[‹]	00E2 20AC 00B9	[‹]	2039
+[›]	00E2 20AC 00BA	[›]	203A
+[“]	00E2 20AC 0153	[“]	201C
+[ ]	00E2 20AC 0160	[ ]	200A
+[‚]	00E2 20AC 0161	[‚]	201A
+[„]	00E2 20AC 017E	[„]	201E
+[ ]	00E2 20AC 0192	[ ]	2003
+[ ]	00E2 20AC 02C6	[ ]	2008
+[‘]	00E2 20AC 02DC	[‘]	2018
+[ ]	00E2 20AC 201A	[ ]	2002
+[–]	00E2 20AC 201C	[–]	2013
+[—]	00E2 20AC 201D	[—]	2014
+[ ]	00E2 20AC 201E	[ ]	2004
+[ ]	00E2 20AC 2020	[ ]	2006
+[ ]	00E2 20AC 2021	[ ]	2007
+[ ]	00E2 20AC 2026	[ ]	2005
+[ ]	00E2 20AC 2030	[ ]	2009
+[​]	00E2 20AC 2039	[​]	200B
+[ ]	00E2 20AC 20AC	[ ]	2000
+[’]	00E2 20AC 2122	[’]	2019
+[â€�]	00E2 20AC FFFD	[”]	201D
+[â� ]	00E2 FFFD 00A0	[⁠]	2060
+[]	00EF 00BB 00BF	[]	FEFF
+[�]	00EF 00BF 00BD	[�]	FFFD
+[￾]	00EF 00BF 00BE	[￾]	FFFE

data/lib/iudex-core/base.rb

@@ -16,7 +16,7 @@
 
 module Iudex
   module Core
-    VERSION = '1.0.0'
+    VERSION = '1.1.0'
 
     LIB_DIR = File.dirname( __FILE__ ) # :nodoc:
   end

data/lib/iudex-core/mojibake.rb

@@ -0,0 +1,73 @@
+#--
+# Copyright (c) 2011 David Kellum
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you
+# may not use this file except in compliance with the License.  You
+# may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.  See the License for the specific language governing
+# permissions and limitations under the License.
+#++
+
+require 'iudex-core'
+require 'java'
+
+module Iudex::Core
+
+  module MojiBake
+    DEFAULT_CONFIG = File.join( File.dirname( __FILE__ ),
+                                '..', '..', 'config', 'mojibake' )
+
+    def self.load_config( file = DEFAULT_CONFIG )
+      regex = nil
+      mojis = []
+      File.open( file ) do |fin|
+        fin.each do |line|
+          case line
+          when %r{^/([^/]+)/$}
+            regex = $1
+          when /^\[.*?\]\s+([0-9A-F ]+)\s+\[.*\]\s+([0-9A-F]+)$/
+            mojis << [ $1.split( ' ' ), $2 ]
+          end
+        end
+      end
+
+      mh = Java::java.util.HashMap.new( 512 )
+      mojis.each do | moji, rpl |
+        mh.put( jstring( moji ), jstring( rpl ) )
+      end
+      [ regex, mh ]
+    end
+
+    private
+
+    def self.jstring( cps )
+      cs = cps.map { |cp| cp.hex }.to_java( :char )
+      Java::java.lang.String.new( cs )
+    end
+
+  end
+
+  module Filters
+    import 'iudex.core.filters.MojiBakeFilter'
+
+    # Re-open iudex.core.filters.MojiBakeFilter to add config file
+    # based initialization.
+    class MojiBakeFilter
+
+      # Alt constructor taking a configuration file in `mojibake -t`
+      # format.
+      def initialize( key, config_file = MojiBake::DEFAULT_CONFIG )
+        super( key, *MojiBake.load_config( config_file ) )
+      end
+
+    end
+
+  end
+
+end

data/lib/iudex-core.rb

@@ -1,3 +1,4 @@
+
 #--
 # Copyright (c) 2008-2011 David Kellum
 #
@@ -31,7 +32,9 @@ module Iudex
 
     import 'iudex.core.ContentKeys'
     import 'iudex.core.ContentSource'
-    import 'iudex.core.VisitExecutor'
+    import 'iudex.core.VisitManager'
+    import 'iudex.core.VisitQueueFactory'
+    import 'iudex.core.VisitQueue'
     import 'iudex.core.VisitURL'
 
     module Filters
@@ -40,10 +43,13 @@ module Iudex
       import 'iudex.core.filters.DateChangeFilter'
       import 'iudex.core.filters.DefaultFilter'
       import 'iudex.core.filters.FutureDateFilter'
-      import 'iudex.core.filters.RLDomainFilter'
+      import 'iudex.core.filters.RedirectHandler'
+      import 'iudex.core.filters.Revisitor'
       import 'iudex.core.filters.TextCtrlWSFilter'
       import 'iudex.core.filters.UHashMDCSetter'
     end
 
   end
 end
+
+require 'iudex-core/mojibake'

data/pom.xml

@@ -5,13 +5,13 @@
   <groupId>iudex</groupId>
   <artifactId>iudex-core</artifactId>
   <packaging>jar</packaging>
-  <version>1.0.0</version>
+  <version>1.1.0</version>
   <name>Iudex Core System</name>
 
   <parent>
     <groupId>iudex</groupId>
     <artifactId>iudex-parent</artifactId>
-    <version>1.0</version>
+    <version>1.1</version>
     <relativePath>..</relativePath>
   </parent>
 
@@ -30,19 +30,19 @@
     <dependency>
       <groupId>iudex</groupId>
       <artifactId>iudex-filter</artifactId>
-      <version>[1.0,1.1)</version>
+      <version>[1.1,1.2)</version>
     </dependency>
 
     <dependency>
       <groupId>iudex</groupId>
       <artifactId>iudex-http</artifactId>
-      <version>[1.0,1.1)</version>
+      <version>[1.1,1.2)</version>
     </dependency>
 
     <dependency>
       <groupId>iudex</groupId>
       <artifactId>iudex-barc</artifactId>
-      <version>[1.0,1.1)</version>
+      <version>[1.1,1.2)</version>
     </dependency>
 
     <dependency>

data/test/test_content_fetcher.rb

@@ -37,10 +37,16 @@ module TestHTTPMocks
   WEAK_ETAG = 'W/"weak-etag"'
 
   class MockSession < Iudex::HTTP::HTTPSession
-    import 'com.gravitext.util.ByteBufferInputStream'
     import 'java.nio.ByteBuffer'
     include Iudex::HTTP
 
+    attr_writer :status
+
+    def initialize
+      super()
+      @status = 200
+    end
+
     def requestHeaders
       [ ]
     end
@@ -49,12 +55,12 @@ module TestHTTPMocks
       [ Header.new( "ETag", WEAK_ETAG ) ]
     end
 
-    def responseCode
-      200
+    def statusCode
+      @status
     end
 
-    def responseStream
-      ByteBufferInputStream.new( ByteBuffer::wrap( "".to_java_bytes ) )
+    def responseBody
+      ByteBuffer::wrap( "BODY".to_java_bytes )
     end
 
     def statusText
@@ -62,7 +68,10 @@ module TestHTTPMocks
     end
 
     def execute( handler )
-      handler.handle_success( self )
+      handler.session_completed( self )
+    end
+
+    def close
     end
   end
 
@@ -88,6 +97,20 @@ module TestHTTPMocks
     end
   end
 
+  import 'iudex.core.VisitCounter'
+
+  class TestVisitCounter
+    include VisitCounter
+    attr_reader :released
+
+    def add( order )
+    end
+
+    def release( acquired, newOrder )
+      @released = acquired.url
+    end
+  end
+
 end
 
 class TestContentFetcher < MiniTest::Unit::TestCase
@@ -119,54 +142,27 @@ class TestContentFetcher < MiniTest::Unit::TestCase
   def test_304
     client = MockHTTPClient.new
     def client.request( session, handler )
-      handler.handle_error( session, 304 )
+      session.status = 304
+      handler.session_completed( session )
     end
     fetch( create_content, client ) do |out|
       assert_equal( DEFAULT_URL, out.url.to_s )
       assert_equal( 304, out.status )
-      assert_nil( out.etag )
-      assert_nil( out.source )
-    end
-  end
-
-  REDIRECT_URL  = "http://gravitext.com/redirect#foo"
-  REDIRECT_NORM = "http://gravitext.com/redirect"
-
-  def test_redirect
-    client = MockHTTPClient.new
-    def client.create_session
-      s = MockSession.new
-      def s.execute( handler )
-        self.url = REDIRECT_URL
-        super
-      end
-      s
-    end
-    fetch( create_content, client ) do |out|
-      assert_equal( REDIRECT_NORM, out.url.to_s )
-      assert_equal( 200, out.status )
-
-      ref = out.referer
-
-      assert_equal( DEFAULT_URL, ref.url.to_s )
-      assert_equal( 302, ref.status )
-      assert_equal( REDIRECT_NORM, ref.referent.url.to_s )
     end
   end
 
   import "java.net.UnknownHostException"
-  import "java.io.IOException"
 
   def test_connect_error
     client = MockHTTPClient.new
     def client.create_session
       s = MockSession.new
       def s.execute( handler )
-        handler.handle_exception( self,
-                                  UnknownHostException.new( "foobar.com" ) )
+        self.error = UnknownHostException.new( "foobar.com" )
+        handler.session_completed( self )
       end
-      def s.responseCode
-        nil
+      def s.statusCode
+        -1
       end
       def s.responseHeaders
         nil
@@ -182,7 +178,9 @@ class TestContentFetcher < MiniTest::Unit::TestCase
 
   def fetch( content, client = MockHTTPClient.new, &block )
     rec = TestReceiver.new( &block )
+    counter = TestVisitCounter.new
     cf = ContentFetcher.new( client,
+                             counter,
                              FilterChain.new( "test-rec", [ rec ] ) )
     cf.filter( content )
   end

data/test/test_content_source.rb

@@ -0,0 +1,75 @@
+#!/usr/bin/env jruby
+#.hashdot.profile += jruby-shortlived
+
+#--
+# Copyright (c) 2011 David Kellum
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you
+# may not use this file except in compliance with the License.  You
+# may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.  See the License for the specific language governing
+# permissions and limitations under the License.
+#++
+
+require File.join( File.dirname( __FILE__ ), "setup" )
+require 'iudex-core'
+
+class TestContentSource < MiniTest::Unit::TestCase
+  include Iudex::Core
+
+  import 'java.nio.ByteBuffer'
+  import 'java.nio.charset.Charset'
+
+  def self.charset( name )
+    Charset::lookup( name )
+  end
+
+  UTF8 = charset( "UTF-8" )
+  ISO1 = charset( "ISO-8859-1" )
+
+  def setup
+    @cs = ContentSource.new( ByteBuffer::wrap( "any".to_java_bytes ) )
+  end
+
+  def test_default_encoding
+    refute( @cs.default_encoding )
+  end
+
+  def test_default_encoding
+    assert( @cs.set_default_encoding( UTF8, 0.0 ) )
+    assert_equal( UTF8, @cs.default_encoding )
+    assert_in_epsilon( 0.0, @cs.encoding_confidence )
+  end
+
+  def test_default_encoding_additive
+    2.times { assert( @cs.set_default_encoding( UTF8, 0.10 ) ) }
+    assert_equal( UTF8, @cs.default_encoding )
+    assert_in_epsilon( 0.20, @cs.encoding_confidence )
+  end
+
+  def test_default_encoding_map
+    assert( @cs.set_default_encoding( { UTF8 => f( 0.10 ),
+                                        ISO1 => f( 0.20 ) } ) )
+    assert_equal( ISO1, @cs.default_encoding )
+    assert_in_epsilon( 0.20, @cs.encoding_confidence )
+
+    refute( @cs.set_default_encoding( {} ) )
+    refute( @cs.set_default_encoding( { UTF8 => f( 0.05 ) } ) )
+    assert( @cs.set_default_encoding( { UTF8 => f( 0.07 ),
+                                        ISO1 => f( 0.01 ) } ) )
+
+    assert_equal( UTF8, @cs.default_encoding )
+    assert_in_epsilon( 0.22, @cs.encoding_confidence )
+  end
+
+  def f( v )
+    Java::java.lang.Float.new( v )
+  end
+
+end

data/test/test_mojibake.rb

@@ -0,0 +1,58 @@
+#!/usr/bin/env jruby
+# -*- coding: utf-8 -*-
+#.hashdot.profile += jruby-shortlived
+
+#--
+# Copyright (c) 2011 David Kellum
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you
+# may not use this file except in compliance with the License.  You
+# may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.  See the License for the specific language governing
+# permissions and limitations under the License.
+#++
+
+require File.join( File.dirname( __FILE__ ), "setup" )
+require 'iudex-core/mojibake'
+
+class TestMojiBake < MiniTest::Unit::TestCase
+  include Gravitext::HTMap
+  include Iudex::Core
+  include Iudex::Core::Filters
+
+  UniMap.define_accessors
+
+  FILTER = MojiBakeFilter.new( ContentKeys::SUMMARY )
+
+  def test_nomatch_recover
+    assert_filter( '', '' )
+    assert_filter( 'ascii', 'ascii' )
+    assert_filter( 'Â', 'Â' )
+  end
+
+  def test_simple_recover
+    assert_filter( '[°]', '[°]' )
+    assert_filter( '“quoted”', '“quotedâ€�' )
+    assert_filter( '“quoted”', '“quoted”' )
+  end
+
+  def test_recursive_recover
+    assert_filter( '°', '°' )
+    assert_filter( 'AP – Greenlake', 'AP – Greenlake' )
+    assert_filter( 'you’re', 'you’re' )
+  end
+
+  def assert_filter( output, input )
+    map = UniMap.new
+    map.summary = input
+    assert( FILTER.filter( map ) )
+    assert_equal( output, map.summary.to_s, "From: #{input}" )
+  end
+
+end