cheap_skate 0.0.1

data/LICENSE

@@ -0,0 +1,16 @@
+CheapSkate: a simple Solr emulator for Ruby
+Copyright (C) 2010  Ross Singer (rossfsinger@gmail.com), Talis (http://talis.com/)
+
+This program is free software; you can redistribute it and/or
+modify it under the terms of the GNU General Public License
+as published by the Free Software Foundation; either version 2
+of the License, or (at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program; if not, write to the Free Software
+Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.

data/README

@@ -0,0 +1,25 @@
+CheapSkate is a Solr emulator intended for situations where the basic functionality of Solr is needed or desired but a Java application server is not feasible (i.e. a cheap, shared webhosting account).  It uses Ferret (a Ruby Lucene clone) for the fulltext indexing and faceting.
+
+CheapSkate is very much a work in progress and development is based on responding to specific needs, rather than really trying to build an actual Solr clone (hence the current lack of XML responseWriter, for example).
+
+Requirements:
+    sinatra
+    ferret
+    hpricot
+    uuid
+    json
+    faster_csv
+    
+Caveats:
+
+CheapSkate isn't really intended to replace Solr.  If you need something that scales to millions of documents or performs replication or does all the neat things Solr does, use Solr.
+
+Faceting causes a serious performance hit on large result sets due to the fact that the facet results require every document in the index to be loaded.
+
+Todos:
+
+Provide field boosting and analyzers both during index as well as query.
+
+Add the MoreLikeThis analyzer from acts_as_ferret.
+
+Add more responseWriters (starting with XML).

data/conf/cheapskate.yml-dist

@@ -0,0 +1,7 @@
+:default
+    :prefix_path: /solr
+    :ferret:
+        :path: db/skate
+    :facet_score_threshold: 0.0
+    :schema: conf/schema.yaml
+  

data/conf/schema.yml-dist

@@ -0,0 +1,290 @@
+schema: 
+  name: example
+  copyFields: 
+  - cat: text
+  - name: text
+  - manu: text
+  - features: text
+  - includes: text
+  - manu: manu_exact
+  uniqueKey: id
+  fields: 
+    price: 
+      stored: true
+      indexed: true
+      type: float
+    cat: 
+      multiValued: true
+      stored: true
+      indexed: true
+      type: text_ws
+      omitNorms: true
+    name: 
+      stored: true
+      indexed: true
+      type: textgen
+    category: 
+      stored: true
+      indexed: true
+      type: textgen
+    popularity: 
+      stored: true
+      indexed: true
+      type: int
+    content_type: 
+      multiValued: true
+      stored: true
+      indexed: true
+      type: string
+    author: 
+      stored: true
+      indexed: true
+      type: textgen
+    comments: 
+      stored: true
+      indexed: true
+      type: text
+    title: 
+      multiValued: true
+      stored: true
+      indexed: true
+      type: text
+    includes: 
+      termOffsets: true
+      stored: true
+      termVectors: true
+      indexed: true
+      type: text
+      termPositions: true
+    text: 
+      multiValued: true
+      stored: false
+      indexed: true
+      type: text
+    weight: 
+      stored: true
+      indexed: true
+      type: float
+    subject: 
+      stored: true
+      indexed: true
+      type: text
+    id: 
+      required: true
+      stored: true
+      indexed: true
+      type: string
+    text_rev: 
+      multiValued: true
+      stored: false
+      indexed: true
+      type: text_rev
+    manu_exact: 
+      stored: false
+      indexed: true
+      type: string
+    links: 
+      multiValued: true
+      stored: true
+      indexed: true
+      type: string
+    features: 
+      multiValued: true
+      stored: true
+      indexed: true
+      type: text
+    sku: 
+      stored: true
+      indexed: true
+      type: textTight
+      omitNorms: true
+    description: 
+      stored: true
+      indexed: true
+      type: text
+    inStock: 
+      stored: true
+      indexed: true
+      type: boolean
+    payloads: 
+      stored: true
+      indexed: true
+      type: payloads
+    last_modified: 
+      stored: true
+      indexed: true
+      type: date
+    manu: 
+      stored: true
+      indexed: true
+      type: textgen
+      omitNorms: true
+    alphaNameSort: 
+      stored: false
+      indexed: true
+      type: alphaOnlySort
+    keywords: 
+      stored: true
+      indexed: true
+      type: textgen
+  version: "1.2"
+  dynamic_fields: 
+    "*_tf": 
+      stored: true
+      indexed: true
+      type: tfloat
+    "*_l": 
+      stored: true
+      indexed: true
+      type: long
+    "*_b": 
+      stored: true
+      indexed: true
+      type: boolean
+    random_*: 
+      type: random
+    "*_ti": 
+      stored: true
+      indexed: true
+      type: tint
+    "*_d": 
+      stored: true
+      indexed: true
+      type: double
+    "*_tdt": 
+      stored: true
+      indexed: true
+      type: tdate
+    "*_tl": 
+      stored: true
+      indexed: true
+      type: tlong
+    "*_f": 
+      stored: true
+      indexed: true
+      type: float
+    "*_pi": 
+      stored: true
+      indexed: true
+      type: pint
+    attr_*: 
+      multiValued: true
+      stored: true
+      indexed: true
+      type: textgen
+    "*_s": 
+      stored: true
+      indexed: true
+      type: string
+    ignored_*: 
+      multiValued: true
+      type: ignored
+    "*_td": 
+      stored: true
+      indexed: true
+      type: tdouble
+    "*_dt": 
+      stored: true
+      indexed: true
+      type: date
+    "*_t": 
+      stored: true
+      indexed: true
+      type: text
+    "*_i": 
+      stored: true
+      indexed: true
+      type: int
+  types: 
+    pint: 
+      :type: :int
+      :index: :untokenized_omit_norms
+    tfloat: 
+      :type: 
+      :index: :untokenized_omit_norms
+    boolean: 
+      :type: :bool
+      :index: :untokenized_omit_norms
+    phonetic: 
+      :type: :text
+    sfloat: 
+      :type: 
+      :index: :untokenized_omit_norms
+    tdate: 
+      :type: 
+      :index: :untokenized_omit_norms
+    binary: 
+      :type: 
+      :index: :untokenized
+    pdouble: 
+      :type: 
+      :index: :untokenized_omit_norms
+    pfloat: 
+      :type: :float
+      :index: :untokenized_omit_norms
+    plong: 
+      :type: 
+      :index: :untokenized_omit_norms
+    lowercase: 
+      :type: :text
+    text: 
+      :type: :text
+    int: 
+      :type: 
+      :index: :untokenized_omit_norms
+    date: 
+      :type: 
+      :index: :untokenized_omit_norms
+    text_rev: 
+      :type: :text
+    slong: 
+      :type: 
+      :index: :untokenized_omit_norms
+    sint: 
+      :type: 
+      :index: :untokenized_omit_norms
+    textgen: 
+      :type: :text
+    text_ws: 
+      :type: :text
+    random: 
+      :type: 
+      :index: :untokenized
+    tdouble: 
+      :type: 
+      :index: :untokenized_omit_norms
+    tlong: 
+      :type: 
+      :index: :untokenized_omit_norms
+    tint: 
+      :type: 
+      :index: :untokenized_omit_norms
+    alphaOnlySort: 
+      :type: :text
+      :index: :omit_norms
+    sdouble: 
+      :type: 
+      :index: :untokenized_omit_norms
+    pdate: 
+      :type: :date
+      :index: :untokenized_omit_norms
+    double: 
+      :type: 
+      :index: :untokenized_omit_norms
+    string: 
+      :type: :string
+      :index: :untokenized_omit_norms
+    ignored: 
+      :type: :string
+      :index: :untokenized
+    payloads: 
+      :type: :text
+    textTight: 
+      :type: :text
+    long: 
+      :type: 
+      :index: :untokenized_omit_norms
+    float: 
+      :type: 
+      :index: :untokenized_omit_norms
+  defaultSearchField: text

data/config.ru-dist

@@ -0,0 +1,19 @@
+require 'rubygems'
+require 'sinatra'
+
+root_dir = File.dirname(__FILE__)
+
+set :environment, :production
+set :configuration, root_dir+"/conf/cheapskate.yml"
+set :site, :kochief
+set :root,  root_dir
+
+set :logging, false
+disable :run
+
+FileUtils.mkdir_p 'log' unless File.exists?('log')
+log = File.new("log/#{Sinatra::Application.site}.log", "a")
+STDOUT.reopen(log)
+STDERR.reopen(log)
+require 'cheap_skate'
+run CheapSkate::Application

data/lib/cheap_skate.rb

@@ -0,0 +1,23 @@
+module CheapSkate
+  $KCODE = 'u' 
+  #stdlib dependencies 
+  require "rubygems"
+  require 'jcode'  
+  require 'yaml'  
+  require 'cgi'  
+  
+  #gem dependencies
+  require 'faster_csv'
+  require "uuid"
+  require 'ferret'
+  require 'json'
+  require 'hpricot'
+  require 'sinatra'
+
+  #project files
+  require File.dirname(__FILE__)+'/cheap_skate/models'
+  require File.dirname(__FILE__)+'/cheap_skate/schema'
+  require File.dirname(__FILE__)+'/cheap_skate/index'
+  require File.dirname(__FILE__)+'/cheap_skate/application'  
+
+end

data/lib/cheap_skate/application.rb

@@ -0,0 +1,106 @@
+module CheapSkate
+  class Application < Sinatra::Base
+
+    configure do
+
+      config = YAML.load_file(Sinatra::Application.configuration)[Sinatra::Application.site]
+
+      i = CheapSkate::Index.new(config[:ferret]||{}, Schema.new_from_config(YAML.load_file(config[:schema])))
+      i.set_fields_from_schema
+      set :index, i
+      set :prefix_path, (config[:prefix_path]||nil)
+      STDOUT.puts "CheapSkate starting with index schema: #{i.schema.name}"
+      STDOUT.puts  "#{i.reader.num_docs} documents currently indexed."
+    end
+  
+    get '/' do
+      "Welcome to CheapSkate"
+    end
+
+
+    get '/select/' do
+      results = select(params)
+      wt = params["wt"] || "json"
+      results.query_time = qtime
+      if wt == "json"
+        content_type 'application/json', :charset => 'utf-8'
+      end
+      results.send("as_#{wt}")
+    end
+
+    post '/select/' do
+      results = select(params)
+      wt = params["wt"] || "json"
+      results.query_time = qtime
+      results.send("as_#{wt}")
+    end
+
+    get '/update/csv/' do
+      csv = CSVLoader.new(params, settings.index)
+      csv.parse
+    out = <<END
+      <?xml version="1.0" encoding="UTF-8"?>
+      <response>
+      <lst name="responseHeader"><int name="status">0</int><int name="QTime">#{qtime}</int></lst>
+      </response>
+END
+    end
+
+    post '/update/' do
+      i = InputDocument.new(request.env["rack.input"].read, settings.index)
+      i.parse
+      wt = params["wt"] || "json"
+      i.query_time = qtime
+      i.send("as_#{wt}")
+    end
+
+    get '/admin/ping/' do
+      '<response>
+      <lst name="responseHeader"><int name="status">0</int><int name="QTime">1</int><lst name="params"><str name="echoParams">all</str><str name="echoParams">all</str><str name="q">solrpingquery</str><str name="qt">standard</str></lst></lst><str name="status">OK</str>
+      </response>'
+    end
+
+    before do
+      if env[:prefix_path]
+        request.path_info.sub!(/^#{env[:prefix_path]}/,'')
+      end
+      unless request.path_info[-1,1] == "/"
+        request.path_info << "/"
+      end
+      @time = Time.now
+    end
+
+    helpers do
+      def select(params)
+        qry = request.env["rack.input"].read
+        if qry.empty?
+          qry = request.env["rack.request.query_string"]
+        end
+
+        parm = CGI.parse(qry)
+        qt = (params['qt'] || "standard")
+        query = settings.index.send("parse_#{qt}_query".to_sym, parm)
+
+        opts = {}
+        opts[:offset] = (params["start"] || 0).to_i
+        opts[:limit] = (params["rows"] || 10).to_i
+        if params["sort"]
+          opts[:sort] = params["sort"]
+          opts[:sort].sub!(/ asc/,"")
+          opts[:sort].sub!(/ desc/,"DESC")
+        end
+        if params["facet"] == "true"
+          query.extend(Facet)
+          query.add_facets_to_query(parm)  
+          query.parse_facet_query(parm)        
+        end
+        results = settings.index.search(query, opts)
+        results
+      end
+  
+      def qtime
+         ((Time.now - @time) * 1000).to_i    
+       end
+    end
+  end
+end

data/lib/cheap_skate/index.rb

@@ -0,0 +1,181 @@
+module CheapSkate
+  class Index < Ferret::Index::Index
+    attr_accessor :schema
+    def initialize(opts={}, schema=CheapSkate::Schema.new)
+      super(opts)
+      @schema = schema
+    end
+    
+    def set_fields_from_schema
+      index_schema_changed = false
+      schema.field_names.each do |fld|
+        f = schema.field_to_field_info(fld)
+        if !field_infos[f.name]
+          self.writer.field_infos << f
+          self.reader.field_infos << f
+          index_schema_changed = true
+        end
+      end
+      puts "Schema has changed" if index_schema_changed
+        
+    end
+    
+    def set_dynamic_field(field)
+      return if field_infos[field]
+
+      return if schema.fields[field]
+
+      dyn_field = nil
+      schema.dynamic_fields.keys.each do |dyn|
+        if dyn =~ /^\*/
+          r = Regexp.new(dyn.sub(/^\*/,".*"))
+        elsif dyn =~ /\*$/
+          r = Regexp.new(dyn.sub(/\*$/,".*"))
+        end
+        unless (field.to_s =~ r).nil?
+          dyn_field = dyn
+          break
+        else
+          puts "Unable to match #{field.to_s} against a dynamic field pattern"
+        end
+      end
+      return unless dyn_field
+      opts = {}
+      if schema.dynamic_fields[dyn_field][:index] == :no
+        opts[:index] = :no
+        opts[:term_vector] = :no
+      elsif schema.field_types[schema.dynamic_fields[dyn_field][:field_type]][:index]
+        opts[:index] = schema.field_types[schema.dynamic_fields[dyn_field][:field_type]][:index]
+      end
+      if schema.dynamic_fields[dyn_field][:stored] == :no
+        opts[:store] = :no
+      end    
+      puts "Adding dynamic field: #{field}"
+      writer.field_infos.add_field(field, opts)
+    end
+        
+    def search(query, opts={})
+      results = ResultSet.new
+      results.offset = opts[:offset]
+      results.limit = opts[:limit]
+      results.query = query.query.to_s
+      if opts[:limit] < 1
+        opts[:limit] = 1
+      end
+      if query.filter
+        opts[:filter] = query.filter
+      end
+      
+      if query.filter_proc
+        if query.query.is_a?(Ferret::Search::MatchAllQuery) && query.filter.nil?
+          get_facets_from_index_terms(query)
+        else
+          opts[:filter_proc] = query.filter_proc
+        end
+      end
+
+      results.total = self.search_each(query.query, opts) do |id, score|
+        results << @schema.typed_document(self[id])
+      end
+      if query.respond_to?(:facet_fields)
+        facets = {}
+        query.facet_fields.each do | facet, values |
+          facets[facet] = values.sort{|a,b| b[1]<=>a[1]}[query.facet_offset, query.facet_limit]
+        end          
+        query.facet_fields = facets
+        if query.facet_queries
+          query.facet_queries.each do |fq|        
+            fq[:results] = search_each(fq.query, :filter=>fq.filter, :limit=>1) {|id,score|}
+          end
+        end
+        results.extend(Facet)
+        results.add_facets_to_results(query)
+      end
+      results
+    end
+    
+    def get_facets_from_index_terms(query)
+      puts query.facet_fields.inspect
+      query.facet_fields.keys.each do |field|
+        field_terms = reader.terms(field)
+        next unless field_terms
+        field_terms.each do |term, count|
+          query.facet_fields[field][term] = count
+        end
+      end
+    end      
+    
+
+    def parse_standard_query(params)
+      or_and = case 
+      when params["q.op"] then [*params["q.op"]].first
+      else schema.default_operator
+      end
+
+      dflt_field = case
+      when params["df"] then [*params["df"]].first
+      else schema.default_field
+      end
+      parser = Ferret::QueryParser.new(:default_field=>dflt_field, :fields=>reader.tokenized_fields, :or_default=>(or_and=="OR"))
+
+      query = CheapSkate::Query.new
+      q = case params["q"].class.name
+      when "Array" then params["q"].first
+      when "String" then params["q"]
+      else nil
+      end
+      if q && !q.empty? && q != "*:*"
+        query.query = parser.parse(q)
+      else
+        query.query = Ferret::Search::MatchAllQuery.new
+      end
+      if params['fq']
+        query.filter = parse_filtered_query(params)
+      end
+      query
+    end
+    
+    def parse_filtered_query(params)
+      or_and = case 
+      when params["q.op"] then [*params["q.op"]].first
+      else schema.default_operator
+      end
+
+      dflt_field = case
+      when params["df"] then [*params["df"]].first
+      else schema.default_field
+      end
+
+      strict_parser = Ferret::QueryParser.new(:default_field=>dflt_field, :fields=>reader.tokenized_fields, :validate_fields=>true, :or_default=>(or_and=="OR"), :handle_parse_errors=>false)
+      bool = Ferret::Search::BooleanQuery.new
+      [*params['fq']].each do |fq|
+        if (filtq = strict_parser.parse(fq) && !filtq.to_s.empty?)
+          bool.add_query(filtq, :must)
+        else
+          (idx, term) = fq.split(":")
+          term.sub!(/^\"/,'').sub!(/\"$/,'')
+          bool.add_query(Ferret::Search::TermQuery.new(idx.to_sym, term), :must)
+        end
+      end
+      unless bool.to_s.empty?
+        return Ferret::Search::QueryFilter.new(bool)
+      end
+      nil
+    end
+    
+
+    def parse_dismax_query(params)
+      parse_standard_query(params)
+    end
+
+    def parse_morelikethis_query(params)
+
+    end
+    
+    def create_document(id=UUID.generate, boost=1.0)
+      d = CheapSkate::Document.new(id, boost)
+      d.index = self
+      d
+    end
+  end
+end

data/lib/cheap_skate/models.rb

@@ -0,0 +1,286 @@
+module CheapSkate
+  class Document < Ferret::Document
+    attr_accessor :index, :doc_id
+    def initialize(doc_id=UUID.generate, boost=1.0)
+      @doc_id = doc_id
+      super(boost)
+    end
+
+    
+    def add_field(key, value)
+      @index.set_dynamic_field(key.to_sym) unless @index.schema.field_names.index(key.to_sym)
+      if value.is_a?(Array)
+        value.each do |v|
+          add_field(key, v)
+        end
+      else      
+        self[key.to_sym] ||= []
+        self[key.to_sym] << value
+      end
+      if copy_fields = @index.schema.copy_fields[key.to_sym]
+        copy_fields.each do |field|
+          add_field(field, value)
+        end
+      end
+    end  
+  end  
+  
+  class ResultSet
+    attr_accessor :total, :docs, :query, :limit, :offset, :facets, :query_time
+    def <<(obj)
+      @docs ||=[]
+      @docs << obj
+    end
+  
+    def to_hash()
+      response = {"responseHeader"=>{"status"=>0, "QTime"=>self.query_time, "params"=>{"q"=>self.query, "version"=>"2.2", "rows"=>self.limit}}}
+      response["response"] = {"numFound"=>self.total, "start"=>self.offset, "docs"=>[]}
+      if self.docs
+        self.docs.each do |doc|
+          response["response"]["docs"] << doc
+        end
+      end
+      
+      response
+    end    
+  
+    def as_ruby
+      response = self.to_hash
+      response["responseHeader"]["wt"] = "ruby"
+      response.inspect
+    end    
+    def as_json
+      response = self.to_hash
+      response["responseHeader"]["wt"] = "json"
+      response.to_json
+    end
+  end
+  
+  module Facet
+    attr_accessor :facet_queries, :facet_limit, :facet_fields, :facet_offset, :facets, :facet_total
+    def add_facets_to_query(params)
+
+      if params['facet.limit'] && !params['facet.limit'].empty?
+        @facet_limit = params['facet.limit'].first.to_i
+      else
+        @facet_limit = 10
+      end
+      if params['facet.offset'] && !params['facet.offset'].empty?
+        @facet_offset = params['facet.offset'].first.to_i
+      else
+        @facet_offset = 0
+      end
+
+      @facet_fields = {}
+      params["facet.field"].each do | field |
+        @facet_fields[field.to_sym] = {}
+      end      
+
+      @filter_proc = lambda do |doc,score,searcher|
+        @facet_fields.keys.each do |field|
+          [*searcher[doc][field]].each do |term|  
+            next if term.nil?
+            @facet_fields[field][term] ||=0
+            @facet_fields[field][term] += 1
+          end
+        end
+      end
+      
+    end
+    
+    def add_facet_query(query, query_string)
+      @facet_queries ||= []
+      @facet_queries << {:query=>query, :results=>0, :query_string=>query_string}
+    end
+    
+    def add_facets_to_results(query)
+      @facet_fields = query.facet_fields
+      @facet_limit = query.facet_limit
+      @facet_queries = query.facet_queries
+      @facet_offset = query.facet_offset
+    end
+    
+    def parse_facet_query(params)     
+     [*params['facet.query']].each do |q|
+        next unless q
+        bool = Ferret::Search::BooleanQuery.new
+      
+ 
+        (idx, term) = q.split(":")
+        term.sub!(/^\"/,'').sub!(/\"$/,'')
+        bool.add_query(Ferret::Search::TermQuery.new(idx.to_sym, term), :must)        
+ 
+        unless bool.to_s.empty?
+          if @filter
+            bool = Ferret::Search::FilteredQuery(bool, @filter)
+          end
+          query.query = @query
+          query.filter = Ferret::Search::QueryFilter.new(bool)
+          add_facet_query(query, q)
+        end
+      end      
+
+    end 
+    
+    def to_hash
+      r = super
+      p = r["responseHeader"]["params"]
+      p["facets"] = "true"
+      p["facet.field"] = @fields
+      p["facet.limit"] = @limit
+      p["facet.offset"] = @offset
+      p["facet.query"] = @query
+      @total = 400;
+      r["facet_counts"] = {"facet_fields"=>@facet_fields, "facet_queries"=>[]}
+      if @facet_queries
+        @facet_queries.each do | fq |
+          r["facet_counts"]["facet_queries"] << [fq[:query_string], fq[:results]]
+        end
+      end
+      r      
+    end   
+    
+  end
+
+
+
+class FacetResponse
+  attr_accessor :query, :limit, :fields, :offset, :facets, :total
+  def initialize
+    @fields = []
+    @facets = {}
+  end
+  
+  def add_facets(r)
+
+  end
+end
+  
+  class InputDocument
+    attr_reader :doc
+    attr_accessor :query_time
+    def initialize(doc, index)
+      doc.sub!(/^[^\<]*/,'')
+      @doc = Hpricot::XML(doc)
+      @index = index
+    end
+  
+    def parse
+      action = @doc.root.name
+      self.send(action)
+    end
+  
+    def add
+      (@doc/'/add/doc').each do |doc|
+        document = @index.create_document
+        (doc/'field').each do |elem|
+          field = elem.attributes['name']
+          value = nil
+          value = elem.inner_html
+          if field and value
+            if field == "id"
+              document[@index.schema.id_field] = value
+            else
+              document.add_field(field, value)
+            end
+          end
+        end      
+        @index << document
+      end
+    end
+  
+    def commit
+      @index.flush
+    end
+  
+    def delete
+      ids = []
+      (@doc/"/delete/id").each do |del|
+        ids << del.inner_html
+      end
+      (@doc/"/delete/query").each do |del|
+        @index.search_each(del.attributes['q'], :limit=>:all) do |id,score|
+          ids << id
+        end      
+      end
+      unless ids.empty?
+        @index.delete(ids)
+      end
+    end
+  
+    def optimize
+      @index.optimize
+    end
+  
+    def to_hash
+      return {"responseHeader"=>{"QTime"=>self.query_time, "status"=>0}}
+    end
+  
+    def as_ruby
+      return to_hash.inspect
+    end
+  
+    def as_json
+      return to_hash.to_json    
+    end
+  end
+
+
+  class CSVLoader
+    attr_reader :fields, :filename, :file, :field_meta
+    def initialize(params, index)
+      @filename = params['stream.file']
+      if @filename
+        @file = open(@filename)
+      end
+      @field_meta = {}
+      params.each_pair do |key, val|
+        next unless key =~ /^f\./
+        (f,field,arg) = key.split(".")
+        @field_meta[field] ||={}
+        @field_meta[field][arg] = val
+      end
+      @index = index
+    end
+  
+    def parse
+      if @file
+        parse_file
+      end
+    end
+  
+    def parse_file
+      @fields = @file.gets.chomp.split(",")
+      documents = []
+      while line = @file.gets
+        line.chomp!
+        doc = @index.create_document
+        # keep track of where we are on the row
+        i = 0
+        FasterCSV.parse_line(line).each do |field|
+          unless field && @fields[i]
+            i+= 1
+            next
+          end
+          if @fields[i] == "id"
+            doc[@index.schema.id_field] = field
+          else
+            if @field_meta[@fields[i]] && @field_meta[@fields[i]]["split"] == "true"
+              field.split(@field_meta[@fields[i]]["separator"]).each do |f|
+                doc.add_field(@fields[i], f.strip)
+              end
+            else
+              doc.add_field(@fields[i], field.strip)
+            end
+          end        
+          i+=1
+        end  
+        @index << doc    
+      end
+    end
+  end
+
+  class Query
+    attr_accessor :parser, :query, :filter, :filter_proc  
+  end
+end

data/lib/cheap_skate/schema.rb

@@ -0,0 +1,227 @@
+require 'rexml/document'
+require 'yaml'
+class CheapSkate::Schema
+  include CheapSkate
+  attr_reader :name, :fields, :config, :field_types, :id_field, :copy_fields, :dynamic_fields, :default_field, :default_operator
+  def self.xml_to_yaml(xml)
+    doc = REXML::Document.new xml
+    y = {"schema"=>{"types"=>{}, "fields"=>{}}}
+    y["schema"]["name"] = doc.root.attributes["name"]
+    y["schema"]["version"] = doc.root.attributes["version"]
+    doc.each_element("/schema/fields/field") do |field|
+      f = {}
+      field.attributes.each do |a,v|
+        next if a == "name"
+        f[a] = case v
+        when "true" then true
+        when "false" then false
+        else v
+        end
+      end
+      y["schema"]["fields"][field.attributes['name']] = f
+    end
+    doc.each_element("/schema/fields/dynamicField") do |dyn_field|
+      f = {}
+      dyn_field.attributes.each do |a,v|
+        next if a == "name"
+        f[a] = case v
+        when "true" then true
+        when "false" then false
+        else v
+        end
+      end
+      y["schema"]["dynamic_fields"] ||= {}
+      y["schema"]["dynamic_fields"][dyn_field.attributes['name']] = f
+    end    
+    doc.each_element("/schema/types/fieldType") do |type|
+      t = {}
+      t[:type] = case type.attributes['class']
+      when "solr.StrField" then :string
+      when "solr.TextField" then :text
+      when "solr.IntField" then :int
+      when "solr.FloatField" then :float
+      when "solr.BoolField" then :bool
+      when "solr.DateField" then :date
+      end
+      if type.attributes['omitNorms'] &&  type.attributes['omitNorms'] == "true"
+        t[:index] = :omit_norms
+      end
+      unless t[:type] == :text
+        if t[:index] == :omit_norms
+          t[:index] = :untokenized_omit_norms
+        else
+          t[:index] = :untokenized
+        end
+      end
+      y["schema"]["types"][type.attributes['name']] = t
+    end
+    doc.each_element("/schema/types/fieldtype") do |type|
+      t = {}
+      t[:type] = case type.attributes['class']
+      when "solr.StrField" then :string
+      when "solr.TextField" then :text
+      when "solr.IntField" then :int
+      when "solr.FloatField" then :float
+      when "solr.BoolField" then :bool
+      when "solr.DateField" then :date
+      end
+      if type.attributes['omitNorms'] &&  type.attributes['omitNorms'] == "true"
+        t[:index] = :omit_norms
+      end
+      unless t[:type] == :text
+        if t[:index] == :omit_norms
+          t[:index] = :untokenized_omit_norms
+        else
+          t[:index] = :untokenized
+        end
+      end
+      y["schema"]["types"][type.attributes['name']] = t
+    end  
+    if dflt = doc.elements['/schema/defaultSearchField']
+      y["schema"]["defaultSearchField"] = dflt.get_text.value if dflt.has_text?
+    end
+    if uniq_key = doc.elements['/schema/uniqueKey']
+      y["schema"]["uniqueKey"] = uniq_key.get_text.value if uniq_key.has_text?
+    end    
+    copy_fields = []
+    doc.each_element("/schema/copyField") do |copy|
+      copy_fields << {copy.attributes['source']=>copy.attributes['dest']}
+    end
+    unless copy_fields.empty?
+      y["schema"]["copyFields"] = copy_fields
+    end
+    y.to_yaml
+  end
+  
+  def self.new_from_config(config_hash)
+    schema = self.new
+    schema.load_from_conf(config_hash)
+    schema
+  end
+  
+  def initialize
+    @copy_fields = {}
+  end
+  
+  def load_from_conf(conf)
+    @fields ={}
+    @field_types ={}
+    @name = conf['schema']['name']
+    conf['schema']['fields'].keys.each do |field|
+      @fields[field.to_sym] = {}
+      fld = conf['schema']['fields'][field]
+      @fields[field.to_sym][:field_type] = fld['type'].to_sym
+      if fld['indexed'] == false
+        @fields[field.to_sym][:index] = :no
+      end
+      if fld['stored'] == false
+        @fields[field.to_sym][:store] = :no
+      end  
+      @fields[field.to_sym][:multi_valued] = fld['multiValued']||false
+    end
+    if conf['schema']['dynamic_fields']
+      conf['schema']['dynamic_fields'].keys.each do |field|
+        @dynamic_fields ||= {}
+        @dynamic_fields[field.to_sym] = {}
+        fld = conf['schema']['dynamic_fields'][field]
+        @dynamic_fields[field.to_sym][:field_type] = fld['type'].to_sym
+        if fld['indexed'] == false
+          @dynamic_fields[field.to_sym][:index] = :no
+        end
+        if fld['stored'] == false
+          @dynamic_fields[field.to_sym][:store] = :no
+        end  
+        @dynamic_fields[field.to_sym][:multi_valued] = fld['multiValued']||false
+      end  
+    end  
+    conf['schema']['types'].keys.each do |type|
+      @field_types[type.to_sym] = conf['schema']['types'][type]
+    end
+    conf['schema']['copyFields'].each do |copy|
+      copy.each_pair do | orig, dest|
+        @copy_fields[orig.to_sym] ||= []
+        @copy_fields[orig.to_sym] << dest.to_sym
+      end
+    end
+    @id_field = (conf['schema']['uniqueKey'] || "id").to_sym
+    @default_field = (conf['schema']['defaultSearchField']||"*").to_sym
+    @default_operator = (conf['schema']['defaultOperator']||"OR")   
+  end
+  
+  def typed_document(lazy_doc)
+    doc = {}
+    lazy_doc.fields.each do |field|
+      [*lazy_doc[field]].each do |fld|
+        if doc[field]
+          doc[field] = [*doc[field]]
+          doc[field] << type_field(field, fld)
+        elsif multi_valued?(field)
+          doc[field] = [type_field(field, fld)]
+        else
+          doc[field] = type_field(field, fld)
+        end
+      end
+    end
+    doc    
+  end
+  
+  def multi_valued?(field)
+    if @fields[field]
+      return @fields[field][:multi_valued]
+    else
+      dyn_field = nil
+      @dynamic_fields.keys.each do |dyn|
+        if dyn =~ /^\*/
+          r = Regexp.new(dyn.sub(/^\*/,".*"))
+        elsif dyn =~ /\*$/
+          r = Regexp.new(dyn.sub(/\*$/,".*"))
+        end
+        if field =~ dyn
+          dyn_field = dyn
+          break
+        end        
+      end
+      return dyn_field[:multi_valued] if dyn_field        
+    end
+    false
+  end
+  
+  def type_field(field_name, value)
+    return value.to_s unless @fields[field_name]
+    val = case @field_types[@fields[field_name][:field_type]][:type]
+    when :string then value.to_s
+    when :text then value.to_s
+    when :int then value.to_i
+    when :float then value.to_f
+    when :date then Date.parse(value)
+    when :bool
+      if value == "true"
+        true
+      else
+        false
+      end
+    else
+      val.to_s
+    end
+    val
+  end
+  
+  def field_names
+    return @fields.keys
+  end
+  
+  def field_to_field_info(field_name)
+    opts = {}
+    if @fields[field_name][:index] == :no
+      opts[:index] = :no
+      opts[:term_vector] = :no
+    elsif @field_types[@fields[field_name][:field_type]][:index]
+      opts[:index] = @field_types[@fields[field_name][:field_type]][:index]
+    end
+    if @fields[field_name][:stored] == :no
+      opts[:store] = :no
+    end
+    Ferret::Index::FieldInfo.new(field_name, opts)
+  end
+  
+end

metadata

@@ -0,0 +1,143 @@
+--- !ruby/object:Gem::Specification 
+name: cheap_skate
+version: !ruby/object:Gem::Version 
+  prerelease: false
+  segments: 
+  - 0
+  - 0
+  - 1
+  version: 0.0.1
+platform: ruby
+authors: 
+- Ross Singer
+autorequire: 
+bindir: bin
+cert_chain: []
+
+date: 2010-05-28 00:00:00 -04:00
+default_executable: 
+dependencies: 
+- !ruby/object:Gem::Dependency 
+  name: fastercsv
+  prerelease: false
+  requirement: &id001 !ruby/object:Gem::Requirement 
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        segments: 
+        - 0
+        version: "0"
+  type: :runtime
+  version_requirements: *id001
+- !ruby/object:Gem::Dependency 
+  name: uuid
+  prerelease: false
+  requirement: &id002 !ruby/object:Gem::Requirement 
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        segments: 
+        - 0
+        version: "0"
+  type: :runtime
+  version_requirements: *id002
+- !ruby/object:Gem::Dependency 
+  name: ferret
+  prerelease: false
+  requirement: &id003 !ruby/object:Gem::Requirement 
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        segments: 
+        - 0
+        version: "0"
+  type: :runtime
+  version_requirements: *id003
+- !ruby/object:Gem::Dependency 
+  name: json
+  prerelease: false
+  requirement: &id004 !ruby/object:Gem::Requirement 
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        segments: 
+        - 0
+        version: "0"
+  type: :runtime
+  version_requirements: *id004
+- !ruby/object:Gem::Dependency 
+  name: hpricot
+  prerelease: false
+  requirement: &id005 !ruby/object:Gem::Requirement 
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        segments: 
+        - 0
+        version: "0"
+  type: :runtime
+  version_requirements: *id005
+- !ruby/object:Gem::Dependency 
+  name: sinatra
+  prerelease: false
+  requirement: &id006 !ruby/object:Gem::Requirement 
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        segments: 
+        - 0
+        version: "0"
+  type: :runtime
+  version_requirements: *id006
+description: A Solr-like interface for situations where running a Java application server is not an option (such as shared web hosting).
+email: rossfsinger@gmail.com
+executables: []
+
+extensions: []
+
+extra_rdoc_files: 
+- LICENSE
+- README
+files: 
+- conf/cheapskate.yml-dist
+- conf/schema.yml-dist
+- config.ru-dist
+- lib/cheap_skate.rb
+- lib/cheap_skate/application.rb
+- lib/cheap_skate/index.rb
+- lib/cheap_skate/models.rb
+- lib/cheap_skate/schema.rb
+- LICENSE
+- README
+has_rdoc: true
+homepage: http://github.com/rsinger/CheapSkate
+licenses: []
+
+post_install_message: 
+rdoc_options: 
+- --charset=UTF-8
+require_paths: 
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement 
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      segments: 
+      - 0
+      version: "0"
+required_rubygems_version: !ruby/object:Gem::Requirement 
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      segments: 
+      - 0
+      version: "0"
+requirements: []
+
+rubyforge_project: 
+rubygems_version: 1.3.6
+signing_key: 
+specification_version: 3
+summary: A very simple Solr emulator in Ruby
+test_files: []
+