infod 0.0.2 → 0.0.3.0

data/infod/{Es/fs.rb → fs.rb}

@@ -13,6 +13,10 @@ class E
   end
   alias_method :no, :node
 
+  def inside
+    node.expand_path.to_s.index(FSbase) == 0
+  end
+
   def siblings
     parent.c
   end
@@ -113,7 +117,6 @@ class E
     if f
       p ? (JSON.parse readFile) : readFile
     else
-      puts "tried to open #{d}"
       nil
     end
   rescue Exception => e
@@ -150,7 +153,7 @@ class Pathname
   end
   
   def deleteNode
-    FileUtils.send file? ? :rm : :rmdir, self
+    FileUtils.send (file?||symlink?) ? :rm : :rmdir, self
     parent.deleteNode if parent.c.empty?
   end
 

data/infod/glob.rb

@@ -0,0 +1,26 @@
+#watch __FILE__
+class E
+
+  # glob :: pattern -> [E]
+  def glob p=""
+    (Pathname.glob d + p).map &:E
+  end
+
+  fn 'set/glob',->d,e=nil,_=nil{
+    p = [d,d.pathSegment].compact.map(&:glob).flatten[0..4e2].compact.partition &:inside
+    p[0] }
+
+  fn 'req/randomFile',->e,r{
+    g = F['set/glob'][e]
+    !g.empty? ? [302, {Location: g[rand g.length].uri}, []] : [404]}
+
+  def docs
+    base = docBase
+    [(base if pathSegment!='/' && base.e),
+     (self if base != self && e && uri[-1]!='/'),
+     base.glob(".{e,html,n3,nt,owl,rdf,ttl,txt}"), # docs
+     ((d? && uri[-1]=='/' && uri.size>1) ? c : []) # trailing slash = children
+    ].flatten.compact
+  end
+
+end

data/infod/graph.rb

@@ -0,0 +1,131 @@
+#watch __FILE__
+class E
+=begin
+  graph resolution is two-pass
+
+  protograph/
+ the first-pass will determine if the second-pass needs to run. an eTag will be derived from the return-value and any graph additions preserved for the next pass. ideal fingerprint sources include filestats, mtime checks, extremely trivial SPARQL queries, SHA160 hashes of in-RAM entities. you can define only a second or first-pass and get default behaviour for the other. for more ideas see <http://tools.ietf.org/html/draft-ietf-httpbis-p4-conditional-25#section-2.3>
+
+  graph/
+   a second-pass might query a CBD (concise-bounded description) from a SPARQL store. infod was originally developed as an alternative to fragility & latency of relying on (large, hard-to-implement, must be running, configured & connectable) SPARQL stores by using the filesystem as much as possible, to experiment with hybrids like "touch" a file on successful POSTs so a store only has to be queried occasionally, and to facilitate simply hooking up bits of Ruby code to names rather than try to shoehorn what you're trying to say into some QueryLang where you're additionally without standard library functions necessitating more roundtrips and latency via marshalling/unmarshalling, parsing, loopback network-abstraction, nascent RDF-via-SPARQL-as-ORM-libs.. but go nuts experimenting w/ graph-handlers for this stuff,,i do..
+
+  triple streams are functions which yield triples
+  s,p - URI in String , o - Literal or URI (object must respond to #uri such as '/path'.E or {'uri' => '/some.n3'}
+  these can be formed into pipelines. the data ingesting/massaging stream-processors in feed.rb are modularized this way
+
+=end
+
+  def fromStream m,*i
+    send(*i) do |s,p,o|
+      m[s] = {'uri' => s} unless m[s].class == Hash 
+      m[s][p] ||= []
+      m[s][p].push o unless m[s][p].member? o
+    end; m
+  end
+
+=begin
+ * stream triples into graph (memory)
+ * import missing resources to store (fs)
+ * behave as normal triplr to caller, with
+   side-effect of import/indexing to knowledgebase
+=end
+  def insertDocs triplr, host, p=[], &b
+    graph = fromStream({},triplr)
+    docs = {}
+    graph.map{|u,r|
+      e = u.E                # resource
+      doc = e.ef             # doc
+      doc.e ||               # exists?
+      (docs[doc.uri] ||= {}  # init doc graph
+       docs[doc.uri][u] = r  # add to graph
+       p.map{|p|             # each indexable property
+         r[p].do{|v|         # values exists?
+           v.map{|o|         # each value
+             e.index p,o}}})}# add to property index 
+    docs.map{|doc,g|
+      d = doc.E
+      if !d.e
+        d.w g, true   # write doc
+        d.roonga host # text index
+        puts "#{doc} < #{g.keys.join ' '}"
+      end}
+    graph.triples &b if b # emit the triples
+    self
+  end
+
+  # default protograph - identity + resource-thunks
+  # Resource, Query, Graph -> graphID
+  fn 'protograph/',->e,q,g{
+     g['#'] = {'uri' => '#'}
+    set = (q['set'] && F['set/'+q['set']] || F['set/'])[e,q,g]
+    if set.empty?
+      g.delete '#'
+    else
+      g['#'][RDFs+'member'] = set
+      g['#'][Type] = E[HTTP+'Response']
+      set.map{|u| g[u.uri] = u } # thunk
+    end
+    F['docsID'][g,q]}
+
+  fn 'set/',->e,q,g{
+    s = []
+    s.concat e.docs
+    e.pathSegment.do{|p| s.concat p.docs }
+    # day-dir hinted pagination
+    e.env['REQUEST_PATH'].match(/(.*?\/)([0-9]{4})\/([0-9]{2})\/([0-9]{2})(.*)/).do{|m|
+      u = g['#']
+      t = ::Date.parse "#{m[2]}-#{m[3]}-#{m[4]}"
+      pp = m[1] + (t-1).strftime('%Y/%m/%d') + m[5]
+      np = m[1] + (t+1).strftime('%Y/%m/%d') + m[5]
+      u[Prev] = {'uri' => pp} if pp.E.e || E['http://' + e.env['SERVER_NAME'] + pp].e
+      u[Next] = {'uri' => np} if np.E.e || E['http://' + e.env['SERVER_NAME'] + np].e }
+    s }
+
+  # fs-derived ID for a resource-set
+  fn 'docsID',->g,q{
+   [q.has_key?('nocache').do{|_|rand},
+     g.sort.map{|u,r|
+       [u, r.respond_to?(:m) && r.m]}].h }
+
+  # default graph (filesystem store)
+  # to use a different default-graph function (w/o patching here, or querystring param), define a GET handler on / (or a subdir),
+  # update configuration such as q['graph'] = 'hexastore' and return false or call #response..
+  fn 'graph/',->e,q,m{
+    # force thunks
+    m.values.map{|r|(r.env e.env).graphFromFile m if r.class == E }
+    # cleanup unexpanded thunks
+    m.delete_if{|u,r|r.class==E}}
+
+  fn 'filter/set',->e,m,r{
+    # filter to RDFs set-members, gone will be:
+    # data about docs containing the data
+    # other fragments in a doc not matching search when indexed per-fragment
+    f = m['#'].do{|c| c[RDFs+'member'].do{|m| m.map &:uri }} || [] # members
+    m.keys.map{|u| m.delete u unless f.member? u}} # trim
+
+  def graphFromFile g={}
+    if !e
+      puts "missing file! "+d
+      return
+    end
+    _ = self
+    triplr = @r.do{|r|
+                    r.q['triplr'].do{|t|
+                          respond_to?(t) && t }} || :triplrMIME
+    unless ext=='e' # native graph-format
+      _ = E '/E/rdf/' + [triplr,uri].h.dive
+      unless _.e && _.m > m;       # up to date?
+        e = {} ; puts "< #{uri}"
+        [:triplrInode,triplr].each{|t| fromStream e, t }
+        _.w e, true
+      end
+    end
+    g.mergeGraph _.r true
+  end
+
+  def graph g={}
+    docs.map{|d|d.graphFromFile g}  # tripleStream -> graph
+    g
+  end
+
+end

data/infod/{Es/grep.rb → grep.rb}

@@ -2,10 +2,10 @@
 class E
 
   fn 'set/grep',->e,q,m{
-    q['i'] ||= true # normal people want case-insensitive, i think - comment to unstick
+    q['i'] ||= true # case-insensitive by default
     q['q'].do{|query|
       [e,e.pathSegment].compact.select(&:e).map{|e|
-        grep = "grep -rl#{q.has_key?('i') && 'i'} #{query.sh} #{e.sh}"  # ;puts grep
+        grep = "grep -rl#{q.has_key?('i') ? 'i' : ''} #{query.sh} #{e.sh}"
         `#{grep}`}.map{|r|r.lines.to_a.map{|r|r.chomp.unpathFs}}.flatten}}
 
   fn 'view/grep',->d,e{
@@ -25,7 +25,7 @@ class E
       p = /#{w.join '.*'}/i
 
       [H.css('/css/search'), H.css('/css/grep'),
-       F['view/search/form'][e.q,e],
+       F['view/'+Search][e.q,e],
        {_: :style, c: c.values.map{|i|
            # word color
            b = rand(16777216)

data/infod/{Es/groonga.rb → groonga.rb}

@@ -1,10 +1,13 @@
 #watch __FILE__
 class E
-#  ruby search-engine & column-store
+
+#  adaptor for ruby text-search-engine & column-store
 #  http://groonga.org/ http://ranguba.org/
 
   # query
   fn 'protograph/roonga',->d,e,m{
+
+    # groonga engine
     ga = E.groonga
 
     # search expression
@@ -13,38 +16,43 @@ class E
     # context
     g = e["context"] || d.env['SERVER_NAME']
 
-    # exec expression
-    r = q ? ga.select{|r|(r['graph'] == g) & r["content"].match(q)} : # expression if exists
-            ga.select{|r| r['graph'] == g} # ordered set (index date-range)
-
-    # offset, size
-    start = e['start'].do{|c| c.to_i.max(r.size - 1).min 0 } || 0
-    c = (e['c']||e['count']).do{|c|c.to_i.max(10000).min(0)} || 8
-
-    # are further results traversible?
-    down = r.size > start+c
-    up   = !(start<=0)
+    begin
+      # execute
+      r = (q && !q.empty?) ? ga.select{|r|(r['graph'] == g) & r["content"].match(q)} : # expression if exists
+        ga.select{|r| r['graph'] == g} # ordered set (index date-range)
 
-    # sort results
-    r = r.sort(e.has_key?('score') ? [["_score"]] : [["time", "descending"]],:offset => start,:limit => c)
+      # offset, size
+      start = e['start'].do{|c| c.to_i.max(r.size - 1).min 0 } || 0
+      c = (e['c']||e['count']).do{|c|c.to_i.max(10000).min(0)} || 8
 
-    # pagination resources
-    m['prev']={'uri' => 'prev','url' => '/search','start' => start + c, 'c' => c} if down
-    m['next']={'uri' => 'next','url' => '/search','start' => start - c, 'c' => c} if up
+      # are further results traversible?
+      down = r.size > start+c
+      up   = !(start<=0)
 
-    # search-result identifiers
-    r = r.map{|r| r['.uri'].E }
+      # sort results
+      r = r.sort(e.has_key?('score') ? [["_score"]] : [["time", "descending"]],:offset => start,:limit => c)
 
-    # fragment identifiers
-    m['frag'] = {'uri' => 'frag', 'res' => r}
+      # results -> graph
+      r = r.map{|r| r['.uri'].E }
+      (r.map &:docs).flatten.uniq.map{|r| m[r.uri] = r.env e}
 
-    # containing documents
-    r.map(&:docs).flatten.uniq.map{|r| m[r.uri] = r.env e}
+      m['#'] = {'uri' => '#',
+        RDFs+'member' => r,
+        Type=>E[HTTP+'Response']}
+      m['#'][Prev]={'uri' => '/search' + {'q' => q, 'start' => start + c, 'c' => c}.qs} if down
+      m['#'][Next]={'uri' => '/search' + {'q' => q, 'start' => start - c, 'c' => c}.qs} if up
+      m['/search'] = {Type => E[Search]}
 
-    # no 404 on 0 results - searchbox view
-    m['/s']={'uri'=>'/s'} if m.empty?
+    rescue Groonga::SyntaxError => x
+      m['/search'] = {Type => E[Search]}
+      m['#'] = {
+        Type => E[COGS+'Exception'],
+        Title => "bad expression",
+        Content => CGI.escapeHTML(x.message)}
+      e['nocache']=true
+    end
 
-    F['docsID'][m]}
+    F['docsID'][m,e]}
 
   def E.groonga
     @groonga ||= (require 'groonga'
@@ -72,6 +80,7 @@ class E
   
   # index resource 
   def roonga graph="global", m = self.graph
+    puts "text #{graph} < #{uri} #{m.keys.join ' '}"
     g = E.groonga          # db
     m.map{|u,i|
       r = g[u] || g.add(u) # create or load entry

data/infod/{H/histogram.rb → histogram.rb}

@@ -1,7 +1,11 @@
 #watch __FILE__
 class E
 
-  fn 'view/histogram',->d,e{
+  fn 'view/histogram',->m,e{
+    e.q['a'].do{|a|Fn 'histogram/main',m,e} ||
+    (Fn 'view/facetSelect',m,e)}
+
+  fn 'histogram/main',->d,e{
 
     # a :: attribute to chart
     a = e.q['a'].do{|e|e.expand} || Date
@@ -10,13 +14,13 @@ class E
     n = e.q['bins'].do{|b| b.to_f.max(999.0).min(1)} || 64.0
     
     # hv :: bin template 
-    v = F['view/'+e.q['hv']]
+    v = F['view/'+(e.q['hv']||'title')]
 
     # construct histogram bins
-    (Fn 'view/histogram/bins',d,a,n).do{|h,m|
+    (Fn 'histogram/bins',d,a,n).do{|h,m|
       
       [H.css('/css/hist'),%w{mu hist}.map{|s|H.js('/js/'+s)},
-       (Fn 'view/histogram/render',h),{style: "width: 100%; height: 5em"},
+       (Fn 'histogram',h),{style: "width: 100%; height: 5em"},
        h.map{|b,r|
          # skip empty bins
          r.empty? ? ' ' :
@@ -27,21 +31,20 @@ class E
           { class: 'histBin b'+b.to_s,
             c: [# label bin
                 {_: :h3, c: from + ' → ' + to },
-                # bin children view
-                (v.(r,e) if v)]})}]}}
+                # bin-scoped view
+                v[r,e]]})}]}}
 
   F['view/h']=F['view/histogram']
 
   # Graph, property, numBins  -> {bin -> Graph}
-  fn 'view/histogram/bins',->m,p,nb{
+  fn 'histogram/bins',->m,p,nb{
     h = {}; bw = 0; min = 0; max = 0
     m.map{|u,r|
       # attribute accessor
       r[p]
     }.flatten.do{|v|
       # values
-      v = v.compact.map{|v|
-       ( p == Date ? v.to_time : v ).to_f}
+      v = v.map{|v| p == Date ? v.to_time : v }.select{|v| v.respond_to? :to_f}.map &:to_f
       max = v.max || 0
       min = v.min || 0
       width = (max-min).do{|w| w.zero? ? 1 : w}
@@ -56,17 +59,21 @@ class E
       # binnable properties
       r[p].do{|v|
         v.each{|v|
-          # bin selector
-          b = (((p == Date ? v.to_time : v).to_f - min) / bw).floor
-
-          # append to bin
-          h[b][u] = r }}}
+          # date handling
+          v = p == Date ? v.to_time : v
+          if v.respond_to? :to_f
+            # bin select
+            b = ((v.to_f - min) / bw).floor
+            # append
+            h[b][u] = r
+          end
+        }}}
 
     # histogram model
     [h, {min: min, max: max, bw: bw}] }
 
-  fn 'view/histogram/render',->h{
-    scale = 255 / h.map{|b,r|r.keys.size}.max.do{|m|m.zero? ? 1 : m}.to_f
+  fn 'histogram',->h{
+    scale = 255 / h.map{|b,r|r.keys.size}.max.do{|m|m.zero? ? 1 : m}.do{|m|m.respond_to?(:to_f) ? m.to_f : 1}
     bins = h.keys.sort
     ['<table class=histogram><tr>',
      bins.map{|b|

data/infod/html.rb

@@ -0,0 +1,231 @@
+#watch __FILE__
+
+# Ruby to HTML
+def H _
+  case _
+  when Hash
+    '<'+(_[:_]||:div).to_s+(_.keys-[:_,:c]).map{|a|
+      ' '+a.to_s+'='+"'"+_[a].to_s.chars.map{|c|{"'"=>'%27','>'=>'%3E','<'=>'%3C'}[c]||c}.join+"'"}.join+'>'+
+      (_[:c] ? (H _[:c]) : '')+
+      (_[:_] == :link ? '' : ('</'+(_[:_]||:div).to_s+'>'))
+  when Array
+    _.map{|n|H n}.join
+  else
+    _.to_s if _
+  end
+end
+
+class H
+
+  def H.[] h; H h end
+
+  def H.js a,inline=false
+    p=a+'.js'
+    inline ? {_: :script, c: p.E.r} :
+    {_: :script, type: "text/javascript", src: p}
+  end
+
+  def H.once e,n,*h
+    return if e[n]
+    e[n]=true
+    h
+  end
+end
+
+class Array
+  def html v=nil
+    map{|e|e.html v}.join ' '
+  end
+end
+
+class Object
+  def html *a
+    name = self.class
+    href = "https://duckduckgo.com/?q=ruby+#{name}"
+    "<a href=#{href}><b>#{name}</b></a>"
+  end
+end
+
+class String
+  def br
+    gsub(/\n/,"<br>\n")
+  end
+  def href name=nil
+    '<a href="'+self+'">' + (name||abbrURI) + '</a>'
+  end
+  def abbrURI
+    sub /(?<scheme>[a-z]+:\/\/)?(?<abbr>.*?)(?<frag>[^#\/]*)$/,
+    '<span class="abbr"><span class="scheme">\k<scheme></span>\k<abbr></span><span class="frag">\k<frag></span>'
+  end
+  def html e=nil
+# CGI.escapeHTML self
+    self
+  end
+end
+
+class Fixnum
+  def html e=nil; H({_: :input, type: :number, value: to_s}) end
+end
+
+class Float
+  def html e=nil; H({_: :input, type: :number, value: to_s}) end
+end
+
+class TrueClass
+  def html e=nil; H({_: :input, type: :checkbox, title: :True, checked: :checked}) end
+end
+
+class FalseClass
+  def html e=nil; H({_: :input, type: :checkbox, title: :False}) end
+end
+
+class Hash
+  def html e={'SERVER_NAME'=>'localhost'}, key=true
+    (keys.size == 1 && has_key?('uri')) ? url.href : 
+      H({_: :table, class: :html, c:
+        map{|k,v|
+          {_: :tr, property: k, c:
+           [({_: :td,
+               c: [{_: :a, name: k, href: (k == 'uri' ? v : k), c: k.to_s.abbrURI}], class: :key} if key),
+             {_: :td,
+               c: (case k
+                   when E::Content
+                     v
+                   when 'uri'
+                     u = v.E
+                     {_: :a, id: u, href: u.url, c: v}
+                   else
+                     v.html e
+                   end), class: :val}]}}})
+  end
+end
+
+class E
+
+  def html *a
+    url.href
+  end
+
+  fn 'view',->d,e{
+    d.values.sort_by{|r| r[Date].do{|d| d[0].to_s} || ''}.reverse.
+    map{|r| Fn 'view/select',r,e }}
+
+  fn 'view/base',->d,e,k=true{
+    [H.once(e,'base',H.css('/css/html')),
+     d.values.map{|v|v.html e,k}]}
+
+  fn 'view/select',->r,e{
+    graph = {r.uri => r}
+    view = F['view/base']
+    # find types, skipping malformed/missing info
+    if r.class == Hash
+      (r[Type].class==Array ? r[Type] : [r[Type]]).do{|types|
+        views = types.map{|t|
+          # discard non-URIs
+          t.uri if t.respond_to? :uri}.
+        compact.map{|t|
+          subtype = t
+          type = subtype.split(/\//)[-2]
+          [F['view/' + subtype],
+          (F['view/' + type] if type)]}.
+        flatten.compact
+        view = views[0] unless views.empty?}
+    end
+    view[graph,e]}
+
+  # multiple views (comma-separated)
+  fn 'view/multi',->d,e{
+    e.q['views'].do{|vs|
+      vs.split(',').map{|v|
+        F['view/'+v].do{|f|f[d,e]}}}}
+
+  # enumerate available views
+  fn 'view/?',->d,e{
+    F.keys.grep(/^view\/(?!application|text\/x-)/).map{|v|
+      v = v[5..-1] # eat selector
+      [{_: :a, href: e['REQUEST_PATH']+e.q.merge({'view'=>v}).qs, c: v},"<br>\n"]}}
+
+  def triplrBlob
+    glob.select(&:f).do{|f|f.map{|r|
+        yield r.uri,Type,E('blob')
+        yield r.uri,Content,r.r}} end
+
+  def triplrHref enc=nil
+    yield uri,Content,(f && read).do{|r|enc ? r.force_encoding(enc).to_utf8 : r}.hrefs
+  end
+
+  def contentURIresolve *f
+    send(*f){|s,p,o|
+      yield s, p, p == Content ?
+      (Nokogiri::HTML.parse o).do{|o|
+        o.css('a').map{|a|
+          if a.has_attribute? 'href'
+            (a.set_attribute 'href', (URI.join s, (a.attr 'href'))) rescue nil
+          end}
+        o.to_s} : o}
+  end
+
+  fn Render+'text/html',->d,e{ u = d['#']||{}
+    titles = d.map{|u,r| r[Title] if r.class==Hash }.flatten.compact
+    view = F['view/'+e.q['view'].to_s] || F['view']
+    H ['<!DOCTYPE html>',{_: :html,
+         c: [{_: :head, c: ['<meta charset="utf-8" />',
+                   {_: :title, c: titles.size==1 ? titles[0] : e.uri},
+                   {_: :link, rel: :icon, href:'/css/misc/favicon.ico'},
+     u[Next].do{|n|{_: :link, rel: :next, href: n.uri}},
+     u[Prev].do{|p|{_: :link, rel: :prev, href: p.uri}}]},
+             {_: :body, c: view[d,e]}]}]}
+
+  # property-selector toolbar - utilizes RDFa view
+  fn 'view/p',->d,e{
+    #TODO fragmentURI scheme for selection-state
+    [H.once(e,'property.toolbar',H.once(e,'p',(H.once e,:mu,H.js('/js/mu')),
+     H.js('/js/p'),
+     H.css('/css/table')),
+     {_: :a, href: '#', c: '-', id: :hideP},
+     {_: :a, href: '#', c: '+', id: :showP},
+     {_: :span, id: 'properties',
+       c: E.graphProperties(d).map{|k|
+         {_: :a, class: :n, href: k, c: k.label+' '}}},
+       {_: :style, id: :pS},
+       {_: :style, id: :lS}),
+     (Fn 'view/'+(e.q['pv']||'table'),d,e)]}
+
+  # table-cell placement on sparse matrix of rows/columns
+  # cal.rb contains an example usage
+  fn 'view/t',->d,e,l=nil,a=nil{
+    layout = e.q['table'] || l
+    if layout
+      [H.once(e,'table',H.css('/css/table')),
+       {_: :table, c:
+         {_: :tbody, c: (Fn 'table/'+layout,d).do{|t|
+             rx = t.keys.max
+             rm = t.keys.min
+             c = t.values.map(&:keys)
+             cm = c.map(&:min).min
+             cx = c.map(&:max).max
+             rm && rx && (rm..rx).map{|r|
+               {_: :tr, c: 
+                 t[r].do{|r|
+                   (cm..cx).map{|c|
+                     r[c].do{|c|
+                       {_: :td, class: :cell, c:(Fn 'view/'+(a||e.q['cellview']||'title'),c,e)}
+                     }||{_: :td}}}}} || ''
+           }}}]
+    else
+      "table= layout arg required"
+    end}
+
+  fn 'view/table',->i,e{[H.css('/css/table'),(Fn 'table',i.values,e)]}
+
+  fn 'table',->es,q=nil{ p = {}
+    es.map{|e|e.respond_to?(:keys) &&
+              e.keys.map{|k|p[k]=true}}
+    keys = p.keys
+    keys.empty? ? es.html :
+    H({_: :table,:class => :tab,
+        c: [{_: :tr, c: keys.map{|k|{_: :th, class: :label, property: k, c: k.abbrURI}}},
+            *es.map{|e|
+              {_: :tr, about: e.uri, c: keys.map{|k| {_: :td, property: k, c: k=='uri' ? e.E.html : e[k].html}}}}]})}
+
+end