infod 0.0.1

data/bin/infod

@@ -0,0 +1,37 @@
+#!/usr/bin/env ruby
+
+case ARGV[0]
+when 'apache'
+  ARGV.shift
+  ENV['apache']='y'
+when /^(lo|local)$/
+  ARGV.shift
+  ARGV.unshift *%w{-p 80 -a 127.0.0.1}
+when 'nginx'
+  ARGV.shift
+  ENV['nginx']='y'
+else
+  ARGV.unshift *%w{-p 80}
+end
+ARGV.unshift *%w{--threaded}
+unless ARGV[-1].match /(start|stop|restart|config|install)/
+  ARGV.push 'start'
+end
+
+%w{infod thin}.map{|l| require l}
+
+module Rack
+  module Adapter
+    def self.guess _
+      :rack
+    end
+    def self.load _
+      Rack::Builder.new {
+        use Rack::Deflater
+        run E
+      }.to_app
+    end
+  end
+end
+
+Thin::Runner.new(ARGV).run!

data/config.ru

@@ -0,0 +1,3 @@
+require 'infod'
+use Rack::Deflater
+run E

data/infod/Es/fs.rb

@@ -0,0 +1,154 @@
+class E
+
+  # POSIX-filesystem index for triples
+  # 
+
+  # index a triple 
+  def index p,o
+    # normalize predicate typeclass (accept URI string or resources) 
+    indexEdit E(p),
+    # literal -> URI conversion
+      (o.class == E ? o : E(p).literal(o)),
+       nil
+  end
+
+  # index a triple - no input-cleanup
+  def indexEdit p,o,a
+    return if @noIndex
+    p.pIndex.noIndex[o,self,a]
+  end
+  def noIndex
+    @noIndex = 1
+    self
+  end
+
+  # subtree traverse
+  fn 'set/subtree',->d,r,m{
+    c =(r['c'].do{|c|c.to_i + 1} || 3).max(100) # one extra for start of next-page
+    o = r['d'] =~ /^a/ ? :asc : :desc           # direction
+    ('/'.E.take c, o, d.uri).do{|s|             # take subtree
+      desc, asc = o == :desc ?                  # orient pagination hints
+      [s.pop, s[0]] : [s[0], s.pop]
+      m['prev'] = {'uri' => 'prev', 'url' => desc.url,'d' => 'desc'}
+      m['next'] = {'uri' => 'next', 'url' => asc.url, 'd' => 'asc'}
+      s }}
+
+  # subtree traverse index on p+o cursor
+  fn 'set/index',->d,r,m,f=:rangePO{
+    (# predicate
+     (f == :rangeP ? d : r['p']).expand.E.
+     # query
+     send f,
+     # count
+     (r['c']&&
+      r['c'].to_i.max(808)+1 || 22),
+     # direction
+     (r['d']&&
+      r['d'].match(/^(a|de)sc$/) &&
+      r['d'].to_sym ||
+      :desc),
+     # offset
+     r['offset'],
+     # object
+     (d if f == :rangePO)
+     ).do{|s|
+      # pagination pointers
+      a,b = s[0], s.size > 1 && s.pop
+      desc,asc = r['d'] && r['d']=='asc' && [a,b]||[b,a]
+      # insert pointers in response-graph
+      m['prev']={'uri' => 'prev','url' => d.url,'d' => 'desc','offset' => desc.uri} if desc
+      m['next']={'uri' => 'next','url' => d.url,'d' => 'asc', 'offset' => asc.uri}  if asc
+      s }}
+  F['set/indexPO']=F['set/index']
+  fn 'set/indexP',->d,r,m{Fn 'set/index',d,r,m,:rangeP}
+
+  # predicate index
+  def pIndex
+    '/index'.E.s self
+  end
+
+  # predicate-object index
+  def poIndex o
+    pIndex.s o
+  end
+ 
+  # predicate-object index lookup
+  def po o
+    pIndex[o.class == E ? o : literal(o)]
+  end
+
+  # range query - predicate
+  def rangeP n=8,d=:desc,s=nil,o=nil
+    puts "rangeP #{uri} count #{n} dir #{d} cursor #{s}"
+    pIndex.subtree(n,d,s).map &:ro
+  end
+
+  # range query - predicate-object
+  def rangePO n=8,d=:desc,s=nil,o
+    puts "rangePO #{uri} #{o} count #{n} dir #{d} cursor #{s}"
+    poIndex(o).subtree(n,d,s).map &:ro
+  end
+
+  # E -> [node]
+  def subtree *a
+    u.take *a
+  end
+
+  # E -> [E]
+  def take *a
+    no.take(*a).map &:E
+  end
+
+end
+
+
+class Pathname
+
+  # take N els from fs tree in sorted, depth-first order
+  def take count=1000, direction=:desc, offset=nil
+
+    # construct offset-path
+    offset = to_s + offset.gsub(/\/+/,'/').E.path if offset
+
+    # in-range indicator
+    ok = false
+
+    # result set
+    set=[]
+
+    # asc/desc operators
+    v,m={asc:      [:id,:>=],
+        desc: [:reverse,:<=]}[direction]
+
+    # visitation function
+    visit=->nodes{
+      
+      # sort nodes in asc or desc order
+      nodes.sort_by(&:to_s).send(v).each{|n|
+        ns = n.to_s
+        # have we got enough nodes?
+        return if 0 >= count
+
+        # continue if
+        (# already in-range
+         ok ||
+         # no offset specified
+         !offset ||
+         # offset satisfies in-range operator
+         (sz = [ns,offset].map(&:size).min
+          ns[0..sz-1].send(m,offset[0..sz-1]))) && (
+         if !(c = n.c).empty? # has children?
+           visit.(c)          # visit children
+         else
+           count = count - 1 # decrement wanted-nodes count
+           set.push n        # add node to result-set
+           ok = true         # iterator is now within range
+        end )}}
+
+    visit.(c) # start 
+
+    # result set
+    set
+  end
+
+end

data/infod/Es/groonga.rb

@@ -0,0 +1,101 @@
+#watch __FILE__
+class E
+
+#  http://groonga.org/ http://ranguba.org/
+#  https://github.com/groonga/groonga
+#  https://github.com/ranguba/rroonga
+  # default DB
+  def E.groonga
+    @groonga ||= (require 'groonga'
+                  E['/E/groonga'].groonga
+                  Groonga["E"] )
+  end
+
+  # load or create groongaDB at URI
+  def groonga
+    return Groonga::Database.open d if e # open db
+    dirname.dir                          # create containing dir
+    Groonga::Database.create(:path => d) # create db
+    Groonga::Schema.define{|s|           # create schema
+      s.create_table("E",:type => :hash,:key_type => "ShortText"){|t|
+        t.short_text "uri"
+        t.short_text "graph"
+        t.text "content"
+        t.time "time" }
+      s.create_table("Bigram",
+                     :type => :patricia_trie,
+                     :key_normalize => true,
+                     :default_tokenizer => "TokenBigram"){|t|
+                                  %w{graph content}.map{|c| t.index("E." + c) }}}
+  end
+  
+  # index resource 
+  def roonga graph="global", m = self.graph
+    g = E.groonga          # db
+    m.map{|u,i|
+      r = g[u] || g.add(u) # create or load entry
+      r.uri = u            # update data
+      r.graph = graph.to_s
+      r.content = i.to_s
+      r.time = i[E::Date][0].to_time
+    }
+    self
+  rescue Exception => x
+    $stderr.puts x,x.message
+  end
+  
+  # remove
+  def unroonga
+    g = E.groonga
+    graph.keys.push(uri).map{|u|g[u].delete}
+  end
+
+  # query
+  fn 'graph/roonga',->d,e,m{
+
+    # db
+    ga = E.groonga
+
+    # search expression
+    q = e['q']
+
+    # context
+    g = e["context"] || d.env['SERVER_NAME']
+
+    # offset
+    start = e['start'].do{|c|c.to_i} || 0
+
+    # number of results
+    c = e['c'].do{|c|c.to_i.max(1000).min(0)} || 8
+
+    # exec expression
+    r = q ? ga.select{|r|(r['graph'] == g) & r["content"].match(q)} : # expression if exists
+            ga.select{|r| r['graph'] == g} # ordered set (index date-range)
+
+    # are further results traversible?
+    down = r.size > start+c
+    up   = !(start<=0)
+
+    # sort results
+    r = r.sort(e.has_key?('score') ? [["_score"]] : [["time", "descending"]],:offset => start,:limit => c)
+
+    # pagination resources
+    m['prev']={'uri' => 'prev','url' => '/search','start' => start + c, 'c' => c} if down
+    m['next']={'uri' => 'next','url' => '/search','start' => start - c, 'c' => c} if up
+
+    # search-result identifiers
+    r = r.map{|r| r['.uri'].E }
+
+    # fragment identifiers
+    m['frag'] = {'uri' => 'frag', 'res' => r}
+
+    # containing documents
+    r.map(&:docs).flatten.uniq.map{|r| m[r.uri] = r.env e}
+
+    # no 404 on 0 results - searchbox view
+    m['/s']={'uri'=>'/s'} if m.empty?
+
+    m # result set
+  }
+  
+end

data/infod/Es/redis.rb

@@ -0,0 +1,3 @@
+class E
+
+end

data/infod/Es/sqlite.rb

@@ -0,0 +1,3 @@
+class E
+
+end

data/infod/Es.rb

@@ -0,0 +1,67 @@
+%w{fs groonga redis}.map{|e|require_relative 'Es/'+e}
+
+class E
+
+  # accumulate a graph recursively along set-membership arc
+  def walk p,m={}
+    graph m # accumulative graph
+    o = []  # resources to visit 
+    o.concat m[uri][p]     # outgoing arc targets
+    o.concat (E p).po self # incoming arc sources
+    o.map{|r|              # walk
+      r.E.walk p,m unless m[r.uri]}
+    m
+  end
+
+  # random leaf
+  def randomLeaf
+    c.empty? && self || c.r.randomLeaf
+  end
+  fn 'set/randomLeaf',->d,e,m{[d.randomLeaf]}
+  fn 'req/randomLeaf',->e,r{[302, {Location: e.randomLeaf.uri},[]]}
+
+  # Graph -> [Predicate]
+  def E.graphProperties g
+    g.values.select{|v|v.respond_to? :keys}.map(&:keys).flatten.uniq
+  end
+
+  fn 'filter/p',->e,m,_{
+    a=Hash[*e['p'].split(/,/).map(&:expand).map{|p|[p,true]}.flatten]
+    m.values.map{|r|
+      r.delete_if{|p,o|!a[p]}}}
+
+  fn 'filter/frag',->e,m,r{
+    f = [r.uri].concat m['frag']['res']
+    m.keys.map{|u|
+      m.delete u unless f.member? u}}
+
+  fn 'filter/basic',->o,m,_{
+    d=m.values
+    o['match'] && (p=o['matchP'].expand
+                   d=d.select{|r|r[p].do{|p|(p.class==Array ? p[0] : p).to_s.match o['match']}})
+    o['min'] && (min=o['min'].to_f
+                 p=o['minP'].expand
+                 d=d.select{|r|r[p].do{|p|(p.class==Array ? p[0] : p).to_f >= min }})
+    o['max'] && (max=o['max'].to_f
+                 p=o['maxP'].expand
+                 d=d.select{|r|r[p].do{|p|(p.class==Array ? p[0] : p).to_f <= max }})
+    o['sort'] && (p=o['sort'].expand
+                       _ = d.partition{|r|r[p]}
+                       d =_[0].sort_by{|r|r[p]}.concat _[1] rescue d)
+    o['sortN'] && (p=o['sortN'].expand
+                       _ = d.partition{|r|r[p]}
+                       d =_[0].sort_by{|r|
+                     (r[p].class==Array && r[p] || [r[p]])[0].do{|d|
+                       d.class==String && d.to_i || d
+                     }
+                   }.concat _[1])
+    o.has_key?('reverse') && d.reverse!
+    m.clear;d.map{|r|m[r['uri']]=r}}
+  
+  def self.filter o,m,r
+    o['filter'].do{|f|f.split(/,/).map{|f|Fn 'filter/'+f,o,m,r}}
+    Fn'filter/basic',o,m,r if o.has_any_key ['reverse','sort','max','min','match']
+    m
+  end
+
+end

data/infod/H.rb

@@ -0,0 +1,29 @@
+def H _
+  case _
+  when Hash then
+    '<'+(_[:_]||:div).to_s+(_.keys-[:_,:c]).map{|a|
+      ' '+a.to_s+'='+"'"+
+      _[a].to_s.hsub({"'"=>'%27',
+                       '>'=>'%3E',
+                       '<'=>'%3C'})+"'"}.join+'>'+
+      (_[:c] ? (H _[:c]) : '')+
+      (_[:_] == :link ? '' : ('</'+(_[:_]||:div).to_s+'>'))
+  when Array then
+    _.map{|n|H n}.join
+  else
+    _.to_s
+  end
+end
+
+class H
+
+  def H.js a,inline=false
+    p=a+'.js'
+    inline ? {_: :script, c: p.E.r} :
+    {_: :script, type: "text/javascript", src: p} end
+
+  def H.once e,n,*h
+    return if e[n]
+    e[n]=true
+    h end
+end

data/infod/K.rb

@@ -0,0 +1,197 @@
+class E
+
+  FSbase   = `pwd`.chomp
+  Prefix   = '/@' # resolver for non-local and non-HTTP URIs
+  S        = '<>' # path separator
+
+  # frequently-used URIs
+  W3    = 'http://www.w3.org/'
+  Purl  = 'http://purl.org/'
+  FOAF  = "http://xmlns.com/foaf/0.1/"
+  SIOC  = 'http://rdfs.org/sioc/ns#'
+  SIOCt = 'http://rdfs.org/sioc/types#'
+  MIMEtype = 'http://www.iana.org/assignments/media-types/'
+  DC       = Purl + 'dc/terms/'
+  Date     = DC   + 'date'
+  Modified = DC   + 'modified'
+  Title    = DC   + 'title'
+  Name     = FOAF + 'name'
+  To       = SIOC + 'addressed_to'
+  Creator  = SIOC + 'has_creator'
+  Content  = SIOC + 'content'
+  Type     = W3   + "1999/02/22-rdf-syntax-ns#type"
+  RDFs     = W3   + '2000/01/rdf-schema#'
+  HTTP     = W3   + '2011/http#'
+  Posix    = W3   + 'ns/posix/'
+  Stat     = Posix + 'stat#'
+  Label    = RDFs + 'label'
+  EXIF     = 'http://www.w3.org/2003/12/exif/ns#'
+  Audio    = 'http://www.semanticdesktop.org/ontologies/nid3/#'
+
+  # file-name extension -> MIME type
+  MIME={
+    aif: 'audio/aif',
+    ans: 'text/ansi',
+    atom: 'application/atom+xml',
+    avi: 'video/avi',
+    e: 'application/json+rdf',
+    coffee: 'text/plain',
+    css: 'text/css',
+    csv: 'text/comma-separated-values',
+    doc: 'application/word',
+    flv: 'video/flv',
+    for: 'application/fortran',
+    gemspec: 'application/ruby',
+    gif: 'image/gif',
+    hs: 'application/haskell',
+    html: 'text/html',
+    ico: 'image/x-ico',
+    jpeg: 'image/jpeg',
+    jpg: 'image/jpeg',
+    js:  'application/javascript',
+    json:  'application/json',
+    log: 'text/log',
+    markdown: 'application/markdown',
+    m4a: 'audio/mp4',
+    md: 'application/markdown',
+    mkv: 'video/matroska',
+    mp3: 'audio/mpeg',
+    mp4: 'video/mp4',
+    mpg: 'video/mpg',
+    n3: 'text/rdf+n3',
+    nfo: 'text/nfo',
+    nt:  'text/ntriples',
+    ntriples:  'text/ntriples',
+    owl: 'application/rdf+xml',
+    pdf: 'application/pdf',
+    png: 'image/png',
+    py:  'application/python',
+    rb:  'application/ruby',
+    ru:  'application/ruby',
+    rdf: 'application/rdf+xml',
+    rtf: 'text/rtf',
+    ssv: 'text/semicolon-separated-values',
+    textile: 'application/textile',
+    tsv: 'text/tab-separated-values',
+    ttl: 'text/turtle',
+    txt: 'text/plain',
+    u: 'application/uri',
+    wav: 'audio/wav',
+    wmv: 'video/wmv',
+    xlsx: 'application/excel',
+  }
+
+  # MIME type -> triplrFn
+  MIMEsource={
+    'application/atom+xml' => [:triplrFeed],
+    'application/markdown' => [:triplrMarkdown],
+    'application/org'      => [:triplrOrg],
+    'application/rdf+xml'  => [:triplrRDFformats,:rdfxml],
+    'application/json'     => [:triplrJSON],
+    'application/pdf'      => [:triplrPDF],
+    'application/textile'  => [:triplrTextile],
+    'application/uri'      => [:triplrUriList],
+    'application/word'     => [:triplrWord],
+    'audio/mp4'            => [:triplrStdOut,'faad -i',Audio],
+    'audio/mpeg'           => [:triplrStdOut,'id3info',Audio,/\((.*?)\)$/],
+    'audio'                => [:triplrStdOut,'sndfile-info',Audio],
+    'image'                => [:triplrStdOut,'exiftool',EXIF],
+    'message/rfc822'       => [:triplrMail],
+    'text/ansi'            => [:triplrANSI],
+    'text/comma-separated-values'=>[:triplrCSV,/,/],
+    'text/html'            => [:triplrRDFformats, :rdfa],
+    'text/log'             => [:triplrLog],
+    'text/nfo'             => [:triplrHref,'cp437'],
+    'text/ntriples'        => [:triplrRDFformats, :ntriples],
+    'text/plain'           => [:triplrHref],
+    'text/rtf'             => [:triplrRTF],
+    'text/semicolon-separated-values'=>[:triplrCSV,/;/],
+    'text/tab-separated-values'=>[:triplrCSV,/\t/],
+    'text/turtle'          => [:triplrRDFformats,:turtle],
+  }
+
+  # MIME type -> formatted content
+     Render='render/'
+  fn Render+'application/ld+json',->d,_=nil{E.renderRDF d, :jsonld}
+  fn Render+'application/rdf+xml',->d,_=nil{E.renderRDF d, :rdfxml}
+  fn Render+'text/ntriples',->d,_=nil{E.renderRDF d, :ntriples}
+  fn Render+'text/turtle',  ->d,_=nil{E.renderRDF d, :turtle}
+  fn Render+'text/rdf+n3',  ->d,_=nil{E.renderRDF d, :n3}
+  fn Render+'text/n3',      ->d,_=nil{E.renderRDF d, :n3}
+
+  # render a view even if requested file exists
+  MIMEcook={
+    'application/atom+xml' => true,
+    'application/markdown' => true,
+    'application/json' => true,
+    'application/json+rdf' => true,
+    'application/org' => true,
+    'application/textile' => true,
+    'application/word' => true,
+    'message/rfc822'=> true,
+    'text/ansi'=>true,
+    'text/log'=>true,
+    'text/nfo'=>true,
+    'text/rtf'=>true,
+  }
+  %w{c ruby haskell php python}.map{|t|
+    %w{application/ text/x-}.map{|m|
+      MIMEcook[m+t] = true
+  }}
+
+  # short -> full URI
+  Abbrev={
+    "dc" => DC,
+    "foaf" => FOAF,
+    "rdf" => W3+"1999/02/22-rdf-syntax-ns#",
+    "rdfs" => RDFs,
+    "sioc" => SIOC,
+    "stat" => Stat,
+  }
+  
+  # expose these literals as a path-name 
+  Literal={}
+   [Purl+'dc/elements/1.1/date',
+    Date,
+    DC+'created',
+    Modified,
+   ].map{|f|Literal[f]=true}
+
+  def mime
+    @mime ||= (# dereferenced symlink
+               f = readlink
+
+               # filename extension
+               x = f.ext.downcase.to_sym
+
+               # directory?
+               if d?
+                 "inode/directory"
+               # local MIME-types table
+               elsif MIME[x]
+#                 puts "found mime for #{x} -> #{MIME[x]}"
+                 MIME[x]
+               # Rack MIME-types table
+               elsif Rack::Mime::MIME_TYPES[t = '.' + x.to_s]
+                 Rack::Mime::MIME_TYPES[t]
+               # procmail uses a prefix not an extension
+               elsif f.base.index('msg.')==0
+                 "message/rfc822"
+               # ask FILE(1)
+               elsif f.e
+                 `file --mime-type -b #{f.sh}`.chomp
+               # default
+               else
+                 "application/octet-stream"
+               end)
+  end
+
+  def == u
+      to_s == u.to_s
+  end
+
+  Nginx = ENV['nginx']
+  Apache = ENV['apache']
+  Version = 'http://web.whats-your.name/www/'
+
+end