rjack-solr 3.6.0.0-java

data/webapp/WEB-INF/web.xml

@@ -0,0 +1,153 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!DOCTYPE web-app PUBLIC "-//Sun Microsystems, Inc.//DTD Web Application 2.3//EN" "http://java.sun.com/dtd/web-app_2_3.dtd">
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements.  See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License.  You may obtain a copy of the License at
+
+     http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+
+<web-app>
+
+  <!-- Uncomment if you are trying to use a Resin version before 3.0.19.
+    Their XML implementation isn't entirely compatible with Xerces.
+    Below are the implementations to use with Sun's JVM.
+  <system-property javax.xml.xpath.XPathFactory=
+             "com.sun.org.apache.xpath.internal.jaxp.XPathFactoryImpl"/>
+  <system-property javax.xml.parsers.DocumentBuilderFactory=
+             "com.sun.org.apache.xerces.internal.jaxp.DocumentBuilderFactoryImpl"/>
+  <system-property javax.xml.parsers.SAXParserFactory=
+             "com.sun.org.apache.xerces.internal.jaxp.SAXParserFactoryImpl"/>
+   -->
+
+  <!-- People who want to hardcode their "Solr Home" directly into the
+       WAR File can set the JNDI property here...
+   -->
+  <!--
+    <env-entry>
+       <env-entry-name>solr/home</env-entry-name>
+       <env-entry-value>/put/your/solr/home/here</env-entry-value>
+       <env-entry-type>java.lang.String</env-entry-type>
+    </env-entry>
+   -->
+   
+  <!-- Any path (name) registered in solrconfig.xml will be sent to that filter -->
+  <filter>
+    <filter-name>SolrRequestFilter</filter-name>
+    <filter-class>org.apache.solr.servlet.SolrDispatchFilter</filter-class>
+    <!-- If you are wiring Solr into a larger web application which controls
+         the web context root, you will probably want to mount Solr under
+         a path prefix (app.war with /app/solr mounted into it, for example).
+         You will need to put this prefix in front of the SolrDispatchFilter
+         url-pattern mapping too (/solr/*), and also on any paths for
+         legacy Solr servlet mappings you may be using.
+         For the admin JSP's to work properly in a path-prefixed configuration,
+         the admin folder containing the JSPs needs to be under the app context root
+         named to match the path-prefix.  For example:
+
+            .war
+               xxx
+                 admin
+                   stats.jsp
+    -->
+    <!--
+    <init-param>
+      <param-name>path-prefix</param-name>
+      <param-value>/xxx</param-value>
+    </init-param>
+    -->
+  </filter>
+
+  <filter-mapping>
+    <!--
+      NOTE: When using multicore, /admin JSP URLs with a core specified
+      such as /solr/coreName/admin/stats.jsp get forwarded by a
+      RequestDispatcher to /solr/admin/stats.jsp with the specified core
+      put into request scope keyed as "org.apache.solr.SolrCore".
+
+      It is unnecessary, and potentially problematic, to have the SolrDispatchFilter
+      configured to also filter on forwards.  Do not configure
+      this dispatcher as <dispatcher>FORWARD</dispatcher>.
+    -->
+    <filter-name>SolrRequestFilter</filter-name>
+    <url-pattern>/*</url-pattern>
+  </filter-mapping>
+
+  <!-- Otherwise it will continue to the old servlets -->
+  
+  <servlet>
+    <servlet-name>SolrServer</servlet-name>
+    <display-name>Solr</display-name>
+    <description>Solr Server</description>
+    <servlet-class>org.apache.solr.servlet.SolrServlet</servlet-class>
+    <load-on-startup>1</load-on-startup>
+  </servlet>
+
+  <servlet>
+    <servlet-name>SolrUpdate</servlet-name>
+    <display-name>SolrUpdate</display-name>
+    <description>Solr Update Handler</description>
+    <servlet-class>org.apache.solr.servlet.SolrUpdateServlet</servlet-class>
+    <load-on-startup>2</load-on-startup>
+  </servlet>
+
+  <servlet>
+    <servlet-name>Logging</servlet-name>
+    <servlet-class>org.apache.solr.servlet.LogLevelSelection</servlet-class>
+  </servlet>
+
+  <!-- @Deprecated -->
+  <servlet>
+    <servlet-name>ping</servlet-name>
+    <jsp-file>/admin/ping.jsp</jsp-file>
+  </servlet>
+
+  <servlet-mapping>
+    <servlet-name>SolrServer</servlet-name>
+    <url-pattern>/select/*</url-pattern>
+  </servlet-mapping>
+
+  <servlet-mapping>
+    <servlet-name>SolrUpdate</servlet-name>
+    <url-pattern>/update/*</url-pattern>
+  </servlet-mapping>
+
+  <servlet-mapping>
+    <servlet-name>Logging</servlet-name>
+    <url-pattern>/admin/logging</url-pattern>
+  </servlet-mapping>
+
+  <!-- @Deprecated -->
+  <servlet-mapping>
+    <servlet-name>ping</servlet-name>
+    <url-pattern>/admin/ping</url-pattern>
+  </servlet-mapping>
+
+  <!-- @Deprecated -->
+  <servlet-mapping>
+    <servlet-name>Logging</servlet-name>
+    <url-pattern>/admin/logging.jsp</url-pattern>
+  </servlet-mapping>
+  
+  <mime-mapping>
+    <extension>.xsl</extension>
+    <!-- per http://www.w3.org/TR/2006/PR-xslt20-20061121/ -->
+    <mime-type>application/xslt+xml</mime-type>
+  </mime-mapping>
+
+  <welcome-file-list>
+    <welcome-file>index.jsp</welcome-file>
+    <welcome-file>index.html</welcome-file>
+  </welcome-file-list>
+
+</web-app>

data/webapp/WEB-INF/weblogic.xml

@@ -0,0 +1,12 @@
+<?xml version='1.0' encoding='UTF-8'?>
+<weblogic-web-app
+    xmlns="http://www.bea.com/ns/weblogic/90"
+    xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+    xsi:schemaLocation="http://www.bea.com/ns/weblogic/90 http://www.bea.com/ns/weblogic/90/weblogic-web-app.xsd">
+
+    <container-descriptor>
+	<filter-dispatched-requests-enabled>false</filter-dispatched-requests-enabled>
+    </container-descriptor>
+
+</weblogic-web-app>
+

data/webapp/admin/_info.jsp

@@ -0,0 +1,120 @@
+<%--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements.  See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License.  You may obtain a copy of the License at
+
+     http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+--%>
+<%@ page import="org.apache.solr.core.SolrConfig,
+                 org.apache.solr.core.SolrCore,
+                 org.apache.solr.schema.IndexSchema,
+                 java.io.File"%>
+<%@ page import="java.net.InetAddress"%>
+<%@ page import="java.io.StringWriter"%>
+<%@ page import="org.apache.solr.core.Config"%>
+<%@ page import="org.apache.solr.common.util.XML"%>
+<%@ page import="org.apache.solr.common.SolrException"%>
+<%@ page import="org.apache.lucene.LucenePackage"%>
+<%@ page import="java.net.UnknownHostException" %>
+
+<%!
+  // only try to figure out the hostname once in a static block so 
+  // we don't have a potentially slow DNS lookup on every admin request
+  static InetAddress addr = null;
+  static String hostname = "unknown";
+  static {
+    try {
+      addr = InetAddress.getLocalHost();
+      hostname = addr.getCanonicalHostName();
+    } catch (UnknownHostException e) {
+      //default to unknown
+    }
+  }
+%>
+
+<%
+  // 
+  SolrCore  core = (SolrCore) request.getAttribute("org.apache.solr.SolrCore");
+  if (core == null) {
+    response.sendError( 404, "missing core name in path" );
+    return;
+  }
+    
+  SolrConfig solrConfig = core.getSolrConfig();
+  int port = request.getServerPort();
+  IndexSchema schema = core.getSchema();
+
+  // enabled/disabled is purely from the point of a load-balancer
+  // and has no effect on local server function.  If there is no healthcheck
+  // configured, don't put any status on the admin pages.
+  String enabledStatus = null;
+  String enabledFile = solrConfig.get("admin/healthcheck/text()",null);
+  boolean isEnabled = false;
+  if (enabledFile!=null) {
+    isEnabled = new File(enabledFile).exists();
+  }
+
+  String collectionName = schema!=null ? schema.getName():"unknown";
+
+  String defaultSearch = "";
+  { 
+    StringWriter tmp = new StringWriter();
+    XML.escapeCharData
+      (solrConfig.get("admin/defaultQuery/text()", ""), tmp);
+    defaultSearch = tmp.toString();
+  }
+
+  String solrImplVersion = "";
+  String solrSpecVersion = "";
+  String luceneImplVersion = "";
+  String luceneSpecVersion = "";
+
+  { 
+    Package p;
+    StringWriter tmp;
+
+    p = SolrCore.class.getPackage();
+
+    tmp = new StringWriter();
+    solrImplVersion = p.getImplementationVersion();
+    if (null != solrImplVersion) {
+      XML.escapeCharData(solrImplVersion, tmp);
+      solrImplVersion = tmp.toString();
+    }
+    tmp = new StringWriter();
+    solrSpecVersion = p.getSpecificationVersion() ;
+    if (null != solrSpecVersion) {
+      XML.escapeCharData(solrSpecVersion, tmp);
+      solrSpecVersion = tmp.toString();
+    }
+  
+    p = LucenePackage.class.getPackage();
+
+    tmp = new StringWriter();
+    luceneImplVersion = p.getImplementationVersion();
+    if (null != luceneImplVersion) {
+      XML.escapeCharData(luceneImplVersion, tmp);
+      luceneImplVersion = tmp.toString();
+    }
+    tmp = new StringWriter();
+    luceneSpecVersion = p.getSpecificationVersion() ;
+    if (null != luceneSpecVersion) {
+      XML.escapeCharData(luceneSpecVersion, tmp);
+      luceneSpecVersion = tmp.toString();
+    }
+  }
+  
+  String cwd=System.getProperty("user.dir");
+  String solrHome= solrConfig.getInstanceDir();
+  
+  boolean cachingEnabled = !solrConfig.getHttpCachingConfig().isNever304(); 
+%>

data/webapp/admin/action.jsp

@@ -0,0 +1,94 @@
+<%@ page contentType="text/html; charset=utf-8" pageEncoding="UTF-8"%>
+<%--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements.  See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License.  You may obtain a copy of the License at
+
+     http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+--%>
+<%@ page import="org.apache.solr.common.util.XML"%>
+<%@ page import="org.apache.solr.core.SolrCore,
+                 org.apache.solr.schema.IndexSchema,
+                 java.io.File,
+                 java.net.InetAddress,
+                 java.net.UnknownHostException"%>
+<%@ page import="java.util.Date"%>
+<%@include file="header.jsp" %>
+<%
+
+  String action = request.getParameter("action");
+  String enableActionStatus = "";
+  boolean isValid = false;
+
+  if (action != null) {
+    // Validate fname
+    if ("Enable".compareTo(action) == 0) isValid = true;
+    if ("Disable".compareTo(action) == 0) isValid = true;
+  }
+  if (isValid) {
+    if ("Enable".compareTo(action) == 0) {
+      try {
+        File enableFile = new File(enabledFile);
+        if (enableFile.createNewFile()) {
+          enableActionStatus += "Enable Succeeded (enable file ";
+          enableActionStatus += enabledFile;
+          enableActionStatus += " created)";
+        } else {
+          enableActionStatus += "Already Enabled";
+        }
+      } catch(Exception e) {
+          enableActionStatus += "Enable Failed: " + e.toString();
+      }
+    }
+    if ("Disable".compareTo(action) == 0) {
+      try {
+        File enableFile = new File(enabledFile);
+        if (enableFile.delete()) {
+          enableActionStatus = "Disable Succeeded (enable file ";
+          enableActionStatus += enabledFile;
+          enableActionStatus += " removed)";
+        } else {
+          enableActionStatus = "Already Disabled";
+        }
+      } catch(Exception e) {
+          enableActionStatus += "Disable Failed: " + e.toString();
+      }
+    }
+  } else {
+    enableActionStatus = "Illegal Action";
+  }
+  // :TODO: might want to rework this so any logging change happens *after*
+  SolrCore.log.info(enableActionStatus);
+%>
+<br clear="all">
+<table>
+  <tr>
+    <td>
+      <H3>Action:</H3>
+    </td>
+    <td>
+      <% XML.escapeCharData(action, out); %><br>
+    </td>
+  </tr>
+  <tr>
+    <td>
+      <H4>Result:</H4>
+    </td>
+    <td>
+      <% XML.escapeCharData(enableActionStatus, out); %><br>
+    </td>
+  </tr>
+</table>
+<br><br>
+    <a href=".">Return to Admin Page</a>
+</body>
+</html>

data/webapp/admin/analysis.jsp

@@ -0,0 +1,496 @@
+<%@ page contentType="text/html; charset=utf-8" pageEncoding="UTF-8"%>
+<%--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements.  See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License.  You may obtain a copy of the License at
+
+     http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+--%>
+<%@ page import="org.apache.lucene.analysis.Analyzer,
+                 org.apache.lucene.util.AttributeSource,
+                 org.apache.lucene.util.Attribute,
+                 org.apache.lucene.analysis.TokenStream,
+                 org.apache.lucene.index.Payload,
+                 org.apache.lucene.analysis.CharReader,
+                 org.apache.lucene.analysis.CharStream,
+                 org.apache.lucene.analysis.tokenattributes.*,
+                 org.apache.lucene.util.AttributeReflector,
+                 org.apache.solr.analysis.CharFilterFactory,
+                 org.apache.solr.analysis.TokenFilterFactory,
+                 org.apache.solr.analysis.TokenizerChain,
+                 org.apache.solr.analysis.TokenizerFactory,
+                 org.apache.solr.schema.FieldType,
+                 org.apache.solr.schema.SchemaField,
+                 org.apache.solr.common.util.XML,
+                 javax.servlet.jsp.JspWriter,java.io.IOException
+                "%>
+<%@ page import="java.io.Reader"%>
+<%@ page import="java.io.StringReader"%>
+<%@ page import="java.util.*"%>
+<%@ page import="java.math.BigInteger" %>
+
+<%-- $Id: analysis.jsp 1095519 2011-04-20 21:30:47Z uschindler $ --%>
+<%-- $Source: /cvs/main/searching/org.apache.solrolarServer/resources/admin/analysis.jsp,v $ --%>
+<%-- $Name:  $ --%>
+
+<%@include file="header.jsp" %>
+
+<%
+  // is name a field name or a type name?
+  String nt = request.getParameter("nt");
+  if (nt==null || nt.length()==0) nt="name"; // assume field name
+  nt = nt.toLowerCase(Locale.ENGLISH).trim();
+  String name = request.getParameter("name");
+  if (name==null || name.length()==0) name="";
+  String val = request.getParameter("val");
+  if (val==null || val.length()==0) val="";
+  String qval = request.getParameter("qval");
+  if (qval==null || qval.length()==0) qval="";
+  String verboseS = request.getParameter("verbose");
+  boolean verbose = verboseS!=null && verboseS.equalsIgnoreCase("on");
+  String qverboseS = request.getParameter("qverbose");
+  boolean qverbose = qverboseS!=null && qverboseS.equalsIgnoreCase("on");
+  String highlightS = request.getParameter("highlight");
+  boolean highlight = highlightS!=null && highlightS.equalsIgnoreCase("on");
+%>
+
+<br clear="all">
+
+<h2>Field Analysis</h2>
+
+<form method="POST" action="analysis.jsp" accept-charset="UTF-8">
+<table>
+<tr>
+  <td>
+  <strong>Field
+          <select name="nt">
+    <option <%= nt.equals("name") ? "selected=\"selected\"" : "" %> >name</option>
+    <option <%= nt.equals("type") ? "selected=\"selected\"" : "" %>>type</option>
+          </select></strong>
+  </td>
+  <td>
+  <input class="std" name="name" type="text" value="<% XML.escapeCharData(name, out); %>">
+  </td>
+</tr>
+<tr>
+  <td>
+  <strong>Field value (Index)</strong>
+  <br/>
+  verbose output
+  <input name="verbose" type="checkbox"
+     <%= verbose ? "checked=\"true\"" : "" %> >
+    <br/>
+  highlight matches
+  <input name="highlight" type="checkbox"
+     <%= highlight ? "checked=\"true\"" : "" %> >
+  </td>
+  <td>
+  <textarea class="std" rows="8" cols="70" name="val"><% XML.escapeCharData(val,out); %></textarea>
+  </td>
+</tr>
+<tr>
+  <td>
+  <strong>Field value (Query)</strong>
+  <br/>
+  verbose output
+  <input name="qverbose" type="checkbox"
+     <%= qverbose ? "checked=\"true\"" : "" %> >
+  </td>
+  <td>
+  <textarea class="std" rows="1" cols="70" name="qval"><% XML.escapeCharData(qval,out); %></textarea>
+  </td>
+</tr>
+<tr>
+
+  <td>
+  </td>
+
+  <td>
+  <input class="stdbutton" type="submit" value="analyze">
+  </td>
+
+</tr>
+</table>
+</form>
+
+
+<%
+  SchemaField field=null;
+
+  if (name!="") {
+    if (nt.equals("name")) {
+      try {
+        field = schema.getField(name);
+      } catch (Exception e) {
+        out.print("<strong>Unknown Field: ");
+        XML.escapeCharData(name, out);
+        out.println("</strong>");
+      }
+    } else {
+       FieldType t = schema.getFieldTypes().get(name);
+       if (null == t) {
+        out.print("<strong>Unknown Field Type: ");
+        XML.escapeCharData(name, out);
+        out.println("</strong>");
+       } else {
+         field = new SchemaField("fakefieldoftype:"+name, t);
+       }
+    }
+  }
+
+  if (field!=null) {
+    HashSet<String> matches = null;
+    if (qval!="" && highlight) {
+      Reader reader = new StringReader(qval);
+      Analyzer analyzer =  field.getType().getQueryAnalyzer();
+      TokenStream tstream = analyzer.reusableTokenStream(field.getName(),reader);
+      CharTermAttribute termAtt = tstream.addAttribute(CharTermAttribute.class);
+      tstream.reset();
+      matches = new HashSet<String>();
+      while (tstream.incrementToken()) {
+        matches.add(termAtt.toString());
+      }
+    }
+
+    if (val!="") {
+      out.println("<h3>Index Analyzer</h3>");
+      doAnalyzer(out, field, val, false, verbose, matches);
+    }
+    if (qval!="") {
+      out.println("<h3>Query Analyzer</h3>");
+      doAnalyzer(out, field, qval, true, qverbose, null);
+    }
+  }
+
+%>
+
+
+</body>
+</html>
+
+
+<%!
+  private static void doAnalyzer(JspWriter out, SchemaField field, String val, boolean queryAnalyser, boolean verbose, Set<String> match) throws Exception {
+
+    FieldType ft = field.getType();
+     Analyzer analyzer = queryAnalyser ?
+             ft.getQueryAnalyzer() : ft.getAnalyzer();
+     if (analyzer instanceof TokenizerChain) {
+       TokenizerChain tchain = (TokenizerChain)analyzer;
+       CharFilterFactory[] cfiltfacs = tchain.getCharFilterFactories();
+       TokenizerFactory tfac = tchain.getTokenizerFactory();
+       TokenFilterFactory[] filtfacs = tchain.getTokenFilterFactories();
+
+       if( cfiltfacs != null ){
+         String source = val;
+         for(CharFilterFactory cfiltfac : cfiltfacs ){
+           CharStream reader = CharReader.get(new StringReader(source));
+           reader = cfiltfac.create(reader);
+           if(verbose){
+             writeHeader(out, cfiltfac.getClass(), cfiltfac.getArgs());
+             source = writeCharStream(out, reader);
+           }
+         }
+       }
+
+       TokenStream tstream = tfac.create(tchain.charStream(new StringReader(val)));
+       List<AttributeSource> tokens = getTokens(tstream);
+       if (verbose) {
+         writeHeader(out, tfac.getClass(), tfac.getArgs());
+       }
+
+       writeTokens(out, tokens, ft, verbose, match);
+
+       for (TokenFilterFactory filtfac : filtfacs) {
+         if (verbose) {
+           writeHeader(out, filtfac.getClass(), filtfac.getArgs());
+         }
+
+         final Iterator<AttributeSource> iter = tokens.iterator();
+         tstream = filtfac.create( new TokenStream(tstream.getAttributeFactory()) {
+           
+           public boolean incrementToken() throws IOException {
+             if (iter.hasNext()) {
+               clearAttributes();
+               AttributeSource token = iter.next();
+               Iterator<Class<? extends Attribute>> atts = token.getAttributeClassesIterator();
+               while (atts.hasNext()) // make sure all att impls in the token exist here
+                 addAttribute(atts.next());
+               token.copyTo(this);
+               return true;
+             } else {
+               return false;
+             }
+           }
+          }
+         );
+         tokens = getTokens(tstream);
+
+         writeTokens(out, tokens, ft, verbose, match);
+       }
+
+     } else {
+       TokenStream tstream = analyzer.reusableTokenStream(field.getName(),new StringReader(val));
+       tstream.reset();
+       List<AttributeSource> tokens = getTokens(tstream);
+       if (verbose) {
+         writeHeader(out, analyzer.getClass(), Collections.EMPTY_MAP);
+       }
+       writeTokens(out, tokens, ft, verbose, match);
+     }
+  }
+
+
+  static List<AttributeSource> getTokens(TokenStream tstream) throws IOException {
+    List<AttributeSource> tokens = new ArrayList<AttributeSource>();
+    tstream.reset();
+    while (tstream.incrementToken()) {
+      tokens.add(tstream.cloneAttributes());
+    }
+    return tokens;
+  }
+
+  private static class ReflectItem {
+    final Class<? extends Attribute> attClass;
+    final String key;
+    final Object value;
+    
+    ReflectItem(Class<? extends Attribute> attClass, String key, Object value) {
+      this.attClass = attClass;
+      this.key = key;
+      this.value = value;
+    }
+  }
+  
+  private static class Tok {
+    final String term;
+    final int pos;
+    final List<ReflectItem> reflected = new ArrayList<ReflectItem>();
+    
+    Tok(AttributeSource token, int pos) {
+      this.term = token.addAttribute(CharTermAttribute.class).toString();
+      this.pos = pos;
+      token.reflectWith(new AttributeReflector() {
+        public void reflect(Class<? extends Attribute> attClass, String key, Object value) {
+          // leave out position and term
+          if (CharTermAttribute.class.isAssignableFrom(attClass))
+            return;
+          if (PositionIncrementAttribute.class.isAssignableFrom(attClass))
+            return;
+          reflected.add(new ReflectItem(attClass, key, value));
+        }
+      });
+    }
+  }
+
+  private static interface TokToStr {
+    public String toStr(Tok o);
+  }
+
+  private static void printRow(JspWriter out, String header, String headerTitle, List<Tok>[] arrLst, TokToStr converter, boolean multival, boolean verbose, Set<String> match) throws IOException {
+    // find the maximum number of terms for any position
+    int maxSz=1;
+    if (multival) {
+      for (List lst : arrLst) {
+        maxSz = Math.max(lst.size(), maxSz);
+      }
+    }
+
+
+    for (int idx=0; idx<maxSz; idx++) {
+      out.println("<tr>");
+      if (idx==0 && verbose) {
+        if (header != null) {
+          out.print("<th NOWRAP rowspan=\""+maxSz+"\"");
+          if (headerTitle != null) {
+            out.print(" title=\"");
+            XML.escapeCharData(headerTitle,out);
+            out.print("\"");
+          }
+          out.print(">");
+          XML.escapeCharData(header,out);
+          out.println("</th>");
+        }
+      }
+
+      for (int posIndex=0; posIndex<arrLst.length; posIndex++) {
+        List<Tok> lst = arrLst[posIndex];
+        if (lst.size() <= idx) continue;
+        if (match!=null && match.contains(lst.get(idx).term)) {
+          out.print("<td class=\"highlight\"");
+        } else {
+          out.print("<td class=\"debugdata\"");
+        }
+
+        // if the last value in the column, use up
+        // the rest of the space via rowspan.
+        if (lst.size() == idx+1 && lst.size() < maxSz) {
+          out.print("rowspan=\""+(maxSz-lst.size()+1)+'"');
+        }
+
+        out.print('>');
+
+        XML.escapeCharData(converter.toStr(lst.get(idx)), out);
+        out.print("</td>");
+      }
+
+      out.println("</tr>");
+    }
+
+  }
+
+  /* this method is totally broken, as no charset involved: new String(byte[]) is crap!
+  static String isPayloadString( Payload p ) {
+    String sp = new String(p.getData());
+    for( int i=0; i < sp.length(); i++ ) {
+      if( !Character.isDefined( sp.charAt(i) ) || Character.isISOControl( sp.charAt(i) ) )
+        return "";
+      }
+    return "(" + sp + ")";
+  }
+  */
+
+  static void writeHeader(JspWriter out, Class clazz, Map<String,String> args) throws IOException {
+    out.print("<h4>");
+    out.print(clazz.getName());
+    XML.escapeCharData("   "+args,out);
+    out.println("</h4>");
+  }
+
+
+
+  // readable, raw, pos, type, start/end
+  static void writeTokens(JspWriter out, List<AttributeSource> tokens, final FieldType ft, boolean verbose, Set<String> match) throws IOException {
+
+    // Use a map to tell what tokens are in what positions
+    // because some tokenizers/filters may do funky stuff with
+    // very large increments, or negative increments.
+    HashMap<Integer,List<Tok>> map = new HashMap<Integer,List<Tok>>();
+    boolean needRaw=false;
+    int pos=0, reflectionCount = -1;
+    for (AttributeSource t : tokens) {
+      String text = t.addAttribute(CharTermAttribute.class).toString();
+      if (!text.equals(ft.indexedToReadable(text))) {
+        needRaw=true;
+      }
+
+      pos += t.addAttribute(PositionIncrementAttribute.class).getPositionIncrement();
+      List lst = map.get(pos);
+      if (lst==null) {
+        lst = new ArrayList(1);
+        map.put(pos,lst);
+      }
+      Tok tok = new Tok(t,pos);
+      // sanity check
+      if (reflectionCount < 0) {
+        reflectionCount = tok.reflected.size();
+      } else {
+        if (reflectionCount != tok.reflected.size())
+          throw new RuntimeException("Should not happen: Number of reflected entries differs for position=" + pos);
+      }
+      lst.add(tok);
+    }
+
+    List<Tok>[] arr = (List<Tok>[])map.values().toArray(new ArrayList[map.size()]);
+
+    // Jetty 6.1.3 miscompiles a generics-enabled version..., without generics:
+    Arrays.sort(arr, new Comparator() {
+      public int compare(Object toks, Object toks1) {
+        return ((List<Tok>)toks).get(0).pos - ((List<Tok>)toks1).get(0).pos;
+      }
+    });
+
+    out.println("<table width=\"auto\" class=\"analysis\" border=\"1\">");
+
+    if (verbose) {
+      printRow(out, "position", "calculated from " + PositionIncrementAttribute.class.getName(), arr, new TokToStr() {
+        public String toStr(Tok t) {
+          return Integer.toString(t.pos);
+        }
+      },false,verbose,null);
+    }
+
+    printRow(out, "term text", CharTermAttribute.class.getName(), arr, new TokToStr() {
+      public String toStr(Tok t) {
+        return ft.indexedToReadable(t.term);
+      }
+    },true,verbose,match);
+
+    if (verbose) {
+      if (needRaw) {
+        printRow(out, "raw text", CharTermAttribute.class.getName(), arr, new TokToStr() {
+        public String toStr(Tok t) {
+            // page is UTF-8, so anything goes.
+            return t.term;
+          }
+        },true,verbose,match);
+      }
+
+      for (int att=0; att < reflectionCount; att++) {
+        final ReflectItem item0 = arr[0].get(0).reflected.get(att);
+        final int i = att;
+        printRow(out, item0.key, item0.attClass.getName(), arr, new TokToStr() {
+          public String toStr(Tok t) {
+            final ReflectItem item = t.reflected.get(i);
+            if (item0.attClass != item.attClass || !item0.key.equals(item.key))
+              throw new RuntimeException("Should not happen: attribute types suddenly change at position=" + t.pos);
+            if (item.value instanceof Payload) {
+              Payload p = (Payload) item.value;
+              if( null != p ) {
+                BigInteger bi = new BigInteger( p.getData() );
+                String ret = bi.toString( 16 );
+                if (ret.length() % 2 != 0) {
+                  // Pad with 0
+                  ret = "0"+ret;
+                }
+                //TODO maybe fix: ret += isPayloadString(p);
+                return ret;
+              }
+              return "";
+            } else {
+              return (item.value != null) ? item.value.toString() : "";
+            }
+          }
+        },true,verbose,null);
+      }
+    }
+    
+    out.println("</table>");
+  }
+
+  static String writeCharStream(JspWriter out, CharStream input) throws IOException {
+    out.println("<table width=\"auto\" class=\"analysis\" border=\"1\">");
+    out.println("<tr>");
+
+    out.print("<th NOWRAP>");
+    XML.escapeCharData("text",out);
+    out.println("</th>");
+
+    final int BUFFER_SIZE = 1024;
+    char[] buf = new char[BUFFER_SIZE];
+    int len = 0;
+    StringBuilder sb = new StringBuilder();
+    do {
+      len = input.read( buf, 0, BUFFER_SIZE );
+      if( len > 0 )
+        sb.append(buf, 0, len);
+    } while( len == BUFFER_SIZE );
+    out.print("<td class=\"debugdata\">");
+    XML.escapeCharData(sb.toString(),out);
+    out.println("</td>");
+    
+    out.println("</tr>");
+    out.println("</table>");
+    return sb.toString();
+  }
+
+%>