infod 0.0.2 → 0.0.3

data/infod/{Es/fs.rb → fs.rb}

@@ -13,6 +13,10 @@ class E
   end
   alias_method :no, :node
 
+  def inside
+    node.expand_path.to_s.index(FSbase) == 0
+  end
+
   def siblings
     parent.c
   end
@@ -113,7 +117,6 @@ class E
     if f
       p ? (JSON.parse readFile) : readFile
     else
-      puts "tried to open #{d}"
       nil
     end
   rescue Exception => e
@@ -150,7 +153,7 @@ class Pathname
   end
   
   def deleteNode
-    FileUtils.send file? ? :rm : :rmdir, self
+    FileUtils.send (file?||symlink?) ? :rm : :rmdir, self
     parent.deleteNode if parent.c.empty?
   end
 

data/infod/glob.rb

@@ -0,0 +1,26 @@
+#watch __FILE__
+class E
+
+  # glob :: pattern -> [E]
+  def glob p=""
+    (Pathname.glob d + p).map &:E
+  end
+
+  fn 'set/glob',->d,e=nil,_=nil{
+    p = [d,d.pathSegment].compact.map(&:glob).flatten[0..4e2].compact.partition &:inside
+    p[0] }
+
+  fn 'req/randomFile',->e,r{
+    g = F['set/glob'][e]
+    !g.empty? ? [302, {Location: g[rand g.length].uri}, []] : [404]}
+
+  def docs
+    base = docBase
+    [(base if pathSegment!='/' && base.e),
+     (self if base != self && e && uri[-1]!='/'),
+     base.glob(".{e,html,n3,nt,owl,rdf,ttl,txt}"), # docs
+     ((d? && uri[-1]=='/' && uri.size>1) ? c : []) # trailing slash = children
+    ].flatten.compact
+  end
+
+end

data/infod/graph.rb

@@ -0,0 +1,123 @@
+#watch __FILE__
+class E
+=begin
+  graph resolution is two-pass
+
+  protograph/
+ the first-pass will determine if the second-pass needs to run. an eTag will be derived from the return-value and any graph additions preserved for the next pass. ideal fingerprint sources include filestats, mtime checks, extremely trivial SPARQL queries, SHA160 hashes of in-RAM entities. you can define only a second or first-pass and get default behaviour for the other. for more implementation ideas see: http://tools.ietf.org/html/draft-ietf-httpbis-p4-conditional-25#section-2.3
+
+  graph/
+   a second-pass might query a CBD (concise-bounded description) from a SPARQL store. info"daemon was originally developed as an alternative to fragility & latency of relying on (large, hard-to-implement, must be running, configured & connectable) SPARQL stores by using the filesystem as much as possible, to experiment with hybrids like "touch" a file on successful POSTs so a store only has to be queried occasionally, and to facilitate simply hooking up bits of Ruby code to names rather than try to shoehorn what you're trying to say into some QueryLang where you're additionally without standard library functions thus necessitating more roundtrips and latency via marshalling/unmarshalling, parsing, loopback network-abstraction, nascent RDF-via-SPARQL-as-ORM-libs.. by all means knock yourself out experimenting w/ defining graph-handlers for this stuff though - i know i do!
+
+  triple streams are functions which yield triples
+  s,p - URI in String , o - Literal or URI (object must respond to #uri such as '/path'.E or {'uri' => '/some.n3'}
+  these can be formed into pipelines. the data ingesting/massaging stream-processors in feed.rb are modularized this way
+
+=end
+
+  def fromStream m,*i
+    send(*i) do |s,p,o|
+      m[s] = {'uri' => s} unless m[s].class == Hash 
+      m[s][p] ||= []
+      m[s][p].push o unless m[s][p].member? o
+    end; m
+  end
+
+=begin
+ * stream triples into graph (memory)
+ * import missing resources to store (fs)
+ * behave as normal triplr to caller, with
+   side-effect of import/indexing to knowledgebase
+=end
+  def insertDocs triplr, h=nil, p=[], &b
+    graph = fromStream({},triplr)
+    graph.map{|u,r| # stream -> graph
+      e = u.E           # resource
+      j = e.ef          # doc
+      j.e ||            # exists?
+      (j.w({u=>r},true) ;puts '< '+u # insert doc
+       p.map{|p|        # each indexable property
+     r[p].do{|v|        # values exists?
+       v.map{|o|        # each value
+        e.index p,o}}}  # property index 
+      e.roonga h if h)} # full-text index
+    graph.triples &b if b # emit the triples
+    self
+  end
+
+  # default protograph - identity + resource-thunks
+  # Resource, Query, Graph -> graphID
+  fn 'protograph/',->e,q,g{
+     g['#'] = {'uri' => '#'}
+    set = (q['set'] && F['set/'+q['set']] || F['set/'])[e,q,g]
+    if set.empty?
+      g.delete '#'
+    else
+      g['#'][RDFs+'member'] = set
+      g['#'][Type] = E[HTTP+'Response']
+      set.map{|u| g[u.uri] = u } # thunk
+    end
+    F['docsID'][g,q]}
+
+  fn 'set/',->e,q,g{
+    s = []
+    s.concat e.docs
+    e.pathSegment.do{|p| s.concat p.docs }
+    # day-dir hinted pagination
+    e.env['REQUEST_PATH'].match(/(.*?\/)([0-9]{4})\/([0-9]{2})\/([0-9]{2})(.*)/).do{|m|
+      u = g['#']
+      t = ::Date.parse "#{m[2]}-#{m[3]}-#{m[4]}"
+      pp = m[1] + (t-1).strftime('%Y/%m/%d') + m[5]
+      np = m[1] + (t+1).strftime('%Y/%m/%d') + m[5]
+      u[Prev] = {'uri' => pp} if pp.E.e || E['http://' + e.env['SERVER_NAME'] + pp].e
+      u[Next] = {'uri' => np} if np.E.e || E['http://' + e.env['SERVER_NAME'] + np].e }
+    s }
+
+  # fs-derived ID for a resource-set
+  fn 'docsID',->g,q{
+   [q.has_key?('nocache').do{|_|rand},
+     g.sort.map{|u,r|
+       [u, r.respond_to?(:m) && r.m]}].h }
+
+  # default graph (filesystem store)
+  # to use a different default-graph function (w/o patching here, or querystring param), define a GET handler on / (or a subdir),
+  # update configuration such as q['graph'] = 'hexastore' and return false or call #response..
+  fn 'graph/',->e,q,m{
+    # force thunks
+    m.values.map{|r|(r.env e.env).graphFromFile m if r.class == E }
+    # cleanup unexpanded thunks
+    m.delete_if{|u,r|r.class==E}}
+
+  fn 'filter/set',->e,m,r{
+    # filter to RDFs set-members, gone will be:
+    # data about docs containing the data
+    # other fragments in a doc not matching search when indexed per-fragment
+    f = m['#'].do{|c| c[RDFs+'member'].do{|m| m.map &:uri }} || [] # members
+    m.keys.map{|u| m.delete u unless f.member? u}} # trim
+
+  def graphFromFile g={}
+    if !e
+      puts "missing file! "+d
+      return
+    end
+    _ = self
+    triplr = @r.do{|r|
+                    r.q['triplr'].do{|t|
+                          respond_to?(t) && t }} || :triplrMIME
+    unless ext=='e' # native graph-format
+      _ = E '/E/rdf/' + [triplr,uri].h.dive
+      unless _.e && _.m > m;       # up to date?
+        e = {} ; puts "< #{uri}"
+        [:triplrInode,triplr].each{|t| fromStream e, t }
+        _.w e, true
+      end
+    end
+    g.mergeGraph _.r true
+  end
+
+  def graph g={}
+    docs.map{|d|d.graphFromFile g}  # tripleStream -> graph
+    g
+  end
+
+end

data/infod/{Es/grep.rb → grep.rb}

@@ -2,10 +2,10 @@
 class E
 
   fn 'set/grep',->e,q,m{
-    q['i'] ||= true # normal people want case-insensitive, i think - comment to unstick
+    q['i'] ||= true # case-insensitive by default
     q['q'].do{|query|
       [e,e.pathSegment].compact.select(&:e).map{|e|
-        grep = "grep -rl#{q.has_key?('i') && 'i'} #{query.sh} #{e.sh}"  # ;puts grep
+        grep = "grep -rl#{q.has_key?('i') ? 'i' : ''} #{query.sh} #{e.sh}"
         `#{grep}`}.map{|r|r.lines.to_a.map{|r|r.chomp.unpathFs}}.flatten}}
 
   fn 'view/grep',->d,e{

data/infod/{Es/groonga.rb → groonga.rb}

@@ -1,10 +1,13 @@
 #watch __FILE__
 class E
-#  ruby search-engine & column-store
+
+#  adaptor for ruby text-search-engine & column-store
 #  http://groonga.org/ http://ranguba.org/
 
   # query
   fn 'protograph/roonga',->d,e,m{
+
+    # groonga engine
     ga = E.groonga
 
     # search expression
@@ -13,38 +16,43 @@ class E
     # context
     g = e["context"] || d.env['SERVER_NAME']
 
-    # exec expression
-    r = q ? ga.select{|r|(r['graph'] == g) & r["content"].match(q)} : # expression if exists
-            ga.select{|r| r['graph'] == g} # ordered set (index date-range)
-
-    # offset, size
-    start = e['start'].do{|c| c.to_i.max(r.size - 1).min 0 } || 0
-    c = (e['c']||e['count']).do{|c|c.to_i.max(10000).min(0)} || 8
-
-    # are further results traversible?
-    down = r.size > start+c
-    up   = !(start<=0)
-
-    # sort results
-    r = r.sort(e.has_key?('score') ? [["_score"]] : [["time", "descending"]],:offset => start,:limit => c)
-
-    # pagination resources
-    m['prev']={'uri' => 'prev','url' => '/search','start' => start + c, 'c' => c} if down
-    m['next']={'uri' => 'next','url' => '/search','start' => start - c, 'c' => c} if up
-
-    # search-result identifiers
-    r = r.map{|r| r['.uri'].E }
-
-    # fragment identifiers
-    m['frag'] = {'uri' => 'frag', 'res' => r}
-
-    # containing documents
-    r.map(&:docs).flatten.uniq.map{|r| m[r.uri] = r.env e}
-
-    # no 404 on 0 results - searchbox view
-    m['/s']={'uri'=>'/s'} if m.empty?
-
-    F['docsID'][m]}
+    begin
+      # execute
+      r = (q && !q.empty?) ? ga.select{|r|(r['graph'] == g) & r["content"].match(q)} : # expression if exists
+        ga.select{|r| r['graph'] == g} # ordered set (index date-range)
+
+      # offset, size
+      start = e['start'].do{|c| c.to_i.max(r.size - 1).min 0 } || 0
+      c = (e['c']||e['count']).do{|c|c.to_i.max(10000).min(0)} || 8
+
+      # are further results traversible?
+      down = r.size > start+c
+      up   = !(start<=0)
+
+      # sort results
+      r = r.sort(e.has_key?('score') ? [["_score"]] : [["time", "descending"]],:offset => start,:limit => c)
+
+      # results -> graph
+      r = r.map{|r| r['.uri'].E }
+      (r.map &:docs).flatten.uniq.map{|r| m[r.uri] = r.env e}
+
+      m['#'] = {'uri' => '#',
+        RDFs+'member' => r,
+        Type=>E[HTTP+'Response']}
+      m['#'][Prev]={'uri' => '/search' + {'q' => q, 'start' => start + c, 'c' => c}.qs} if down
+      m['#'][Next]={'uri' => '/search' + {'q' => q, 'start' => start - c, 'c' => c}.qs} if up
+      m['/search'] = {Type => E[Search]}
+
+    rescue Groonga::SyntaxError => x
+      m['/search'] = {Type => E[Search]}
+      m['#'] = {
+        Type => E[COGS+'Exception'],
+        Title => "bad expression",
+        Content => CGI.escapeHTML(x.message)}
+      e['nocache']=true
+    end
+
+    F['docsID'][m,e]}
 
   def E.groonga
     @groonga ||= (require 'groonga'

data/infod/{H/histogram.rb → histogram.rb}

@@ -1,7 +1,11 @@
 #watch __FILE__
 class E
 
-  fn 'view/histogram',->d,e{
+  fn 'view/histogram',->m,e{
+    e.q['a'].do{|a|Fn 'histogram/main',m,e} ||
+    (Fn 'view/facetSelect',m,e)}
+
+  fn 'histogram/main',->d,e{
 
     # a :: attribute to chart
     a = e.q['a'].do{|e|e.expand} || Date
@@ -10,13 +14,13 @@ class E
     n = e.q['bins'].do{|b| b.to_f.max(999.0).min(1)} || 64.0
     
     # hv :: bin template 
-    v = F['view/'+e.q['hv']]
+    v = F['view/'+(e.q['hv']||'title')]
 
     # construct histogram bins
-    (Fn 'view/histogram/bins',d,a,n).do{|h,m|
+    (Fn 'histogram/bins',d,a,n).do{|h,m|
       
       [H.css('/css/hist'),%w{mu hist}.map{|s|H.js('/js/'+s)},
-       (Fn 'view/histogram/render',h),{style: "width: 100%; height: 5em"},
+       (Fn 'histogram',h),{style: "width: 100%; height: 5em"},
        h.map{|b,r|
          # skip empty bins
          r.empty? ? ' ' :
@@ -27,21 +31,20 @@ class E
           { class: 'histBin b'+b.to_s,
             c: [# label bin
                 {_: :h3, c: from + ' → ' + to },
-                # bin children view
-                (v.(r,e) if v)]})}]}}
+                # bin-scoped view
+                v[r,e]]})}]}}
 
   F['view/h']=F['view/histogram']
 
   # Graph, property, numBins  -> {bin -> Graph}
-  fn 'view/histogram/bins',->m,p,nb{
+  fn 'histogram/bins',->m,p,nb{
     h = {}; bw = 0; min = 0; max = 0
     m.map{|u,r|
       # attribute accessor
       r[p]
     }.flatten.do{|v|
       # values
-      v = v.compact.map{|v|
-       ( p == Date ? v.to_time : v ).to_f}
+      v = v.map{|v| p == Date ? v.to_time : v }.select{|v| v.respond_to? :to_f}.map &:to_f
       max = v.max || 0
       min = v.min || 0
       width = (max-min).do{|w| w.zero? ? 1 : w}
@@ -56,17 +59,21 @@ class E
       # binnable properties
       r[p].do{|v|
         v.each{|v|
-          # bin selector
-          b = (((p == Date ? v.to_time : v).to_f - min) / bw).floor
-
-          # append to bin
-          h[b][u] = r }}}
+          # date handling
+          v = p == Date ? v.to_time : v
+          if v.respond_to? :to_f
+            # bin select
+            b = ((v.to_f - min) / bw).floor
+            # append
+            h[b][u] = r
+          end
+        }}}
 
     # histogram model
     [h, {min: min, max: max, bw: bw}] }
 
-  fn 'view/histogram/render',->h{
-    scale = 255 / h.map{|b,r|r.keys.size}.max.do{|m|m.zero? ? 1 : m}.to_f
+  fn 'histogram',->h{
+    scale = 255 / h.map{|b,r|r.keys.size}.max.do{|m|m.zero? ? 1 : m}.do{|m|m.respond_to?(:to_f) ? m.to_f : 1}
     bins = h.keys.sort
     ['<table class=histogram><tr>',
      bins.map{|b|

data/infod/html.rb

@@ -0,0 +1,231 @@
+#watch __FILE__
+
+# Ruby to HTML
+def H _
+  case _
+  when Hash
+    '<'+(_[:_]||:div).to_s+(_.keys-[:_,:c]).map{|a|
+      ' '+a.to_s+'='+"'"+_[a].to_s.chars.map{|c|{"'"=>'%27','>'=>'%3E','<'=>'%3C'}[c]||c}.join+"'"}.join+'>'+
+      (_[:c] ? (H _[:c]) : '')+
+      (_[:_] == :link ? '' : ('</'+(_[:_]||:div).to_s+'>'))
+  when Array
+    _.map{|n|H n}.join
+  else
+    _.to_s if _
+  end
+end
+
+class H
+
+  def H.[] h; H h end
+
+  def H.js a,inline=false
+    p=a+'.js'
+    inline ? {_: :script, c: p.E.r} :
+    {_: :script, type: "text/javascript", src: p}
+  end
+
+  def H.once e,n,*h
+    return if e[n]
+    e[n]=true
+    h
+  end
+end
+
+class Array
+  def html v=nil
+    map{|e|e.html v}.join ' '
+  end
+end
+
+class Object
+  def html *a
+    name = self.class
+    href = "https://duckduckgo.com/?q=ruby+#{name}"
+    "<a href=#{href}><b>#{name}</b></a>"
+  end
+end
+
+class String
+  def br
+    gsub(/\n/,"<br>\n")
+  end
+  def href name=nil
+    '<a href="'+self+'">' + (name||abbrURI) + '</a>'
+  end
+  def abbrURI
+    sub /(?<scheme>[a-z]+:\/\/)?(?<abbr>.*?)(?<frag>[^#\/]*)$/,
+    '<span class="abbr"><span class="scheme">\k<scheme></span>\k<abbr></span><span class="frag">\k<frag></span>'
+  end
+  def html e=nil
+# CGI.escapeHTML self
+    self
+  end
+end
+
+class Fixnum
+  def html e=nil; H({_: :input, type: :number, value: to_s}) end
+end
+
+class Float
+  def html e=nil; H({_: :input, type: :number, value: to_s}) end
+end
+
+class TrueClass
+  def html e=nil; H({_: :input, type: :checkbox, title: :True, checked: :checked}) end
+end
+
+class FalseClass
+  def html e=nil; H({_: :input, type: :checkbox, title: :False}) end
+end
+
+class Hash
+  def html e={'SERVER_NAME'=>'localhost'}, key=true
+    (keys.size == 1 && has_key?('uri')) ? url.href : 
+      H({_: :table, class: :html, c:
+        map{|k,v|
+          {_: :tr, property: k, c:
+           [({_: :td,
+               c: [{_: :a, name: k, href: (k == 'uri' ? v : k), c: k.to_s.abbrURI}], class: :key} if key),
+             {_: :td,
+               c: (case k
+                   when E::Content
+                     v
+                   when 'uri'
+                     u = v.E
+                     {_: :a, id: u, href: u.url, c: v}
+                   else
+                     v.html e
+                   end), class: :val}]}}})
+  end
+end
+
+class E
+
+  def html *a
+    url.href
+  end
+
+  fn 'view',->d,e{
+    d.values.sort_by{|r| r[Date].do{|d| d[0].to_s} || ''}.reverse.
+    map{|r| Fn 'view/select',r,e }}
+
+  fn 'view/base',->d,e,k=true{
+    [H.once(e,'base',H.css('/css/html')),
+     d.values.map{|v|v.html e,k}]}
+
+  fn 'view/select',->r,e{
+    graph = {r.uri => r}
+    view = F['view/base']
+    # find types, skipping malformed/missing info
+    if r.class == Hash
+      (r[Type].class==Array ? r[Type] : [r[Type]]).do{|types|
+        views = types.map{|t|
+          # discard non-URIs
+          t.uri if t.respond_to? :uri}.
+        compact.map{|t|
+          subtype = t
+          type = subtype.split(/\//)[-2]
+          [F['view/' + subtype],
+          (F['view/' + type] if type)]}.
+        flatten.compact
+        view = views[0] unless views.empty?}
+    end
+    view[graph,e]}
+
+  # multiple views (comma-separated)
+  fn 'view/multi',->d,e{
+    e.q['views'].do{|vs|
+      vs.split(',').map{|v|
+        F['view/'+v].do{|f|f[d,e]}}}}
+
+  # enumerate available views
+  fn 'view/?',->d,e{
+    F.keys.grep(/^view\/(?!application|text\/x-)/).map{|v|
+      v = v[5..-1] # eat selector
+      [{_: :a, href: e['REQUEST_PATH']+e.q.merge({'view'=>v}).qs, c: v},"<br>\n"]}}
+
+  def triplrBlob
+    glob.select(&:f).do{|f|f.map{|r|
+        yield r.uri,Type,E('blob')
+        yield r.uri,Content,r.r}} end
+
+  def triplrHref enc=nil
+    yield uri,Content,(f && read).do{|r|enc ? r.force_encoding(enc).to_utf8 : r}.hrefs
+  end
+
+  def contentURIresolve *f
+    send(*f){|s,p,o|
+      yield s, p, p == Content ?
+      (Nokogiri::HTML.parse o).do{|o|
+        o.css('a').map{|a|
+          if a.has_attribute? 'href'
+            (a.set_attribute 'href', (URI.join s, (a.attr 'href'))) rescue nil
+          end}
+        o.to_s} : o}
+  end
+
+  fn Render+'text/html',->d,e{ u = d['#']||{}
+    titles = d.map{|u,r| r[Title] if r.class==Hash }.flatten.compact
+    view = F['view/'+e.q['view'].to_s] || F['view']
+    H ['<!DOCTYPE html>',{_: :html,
+         c: [{_: :head, c: ['<meta charset="utf-8" />',
+                   {_: :title, c: titles.size==1 ? titles[0] : e.uri},
+                   {_: :link, rel: :icon, href:'/css/misc/favicon.ico'},
+     u[Next].do{|n|{_: :link, rel: :next, href: n.uri}},
+     u[Prev].do{|p|{_: :link, rel: :prev, href: p.uri}}]},
+             {_: :body, c: view[d,e]}]}]}
+
+  # property-selector toolbar - utilizes RDFa view
+  fn 'view/p',->d,e{
+    #TODO fragmentURI scheme for selection-state
+    [H.once(e,'property.toolbar',H.once(e,'p',(H.once e,:mu,H.js('/js/mu')),
+     H.js('/js/p'),
+     H.css('/css/table')),
+     {_: :a, href: '#', c: '-', id: :hideP},
+     {_: :a, href: '#', c: '+', id: :showP},
+     {_: :span, id: 'properties',
+       c: E.graphProperties(d).map{|k|
+         {_: :a, class: :n, href: k, c: k.label+' '}}},
+       {_: :style, id: :pS},
+       {_: :style, id: :lS}),
+     (Fn 'view/'+(e.q['pv']||'table'),d,e)]}
+
+  # table-cell placement on sparse matrix of rows/columns
+  # cal.rb contains an example usage
+  fn 'view/t',->d,e,l=nil,a=nil{
+    layout = e.q['table'] || l
+    if layout
+      [H.once(e,'table',H.css('/css/table')),
+       {_: :table, c:
+         {_: :tbody, c: (Fn 'table/'+layout,d).do{|t|
+             rx = t.keys.max
+             rm = t.keys.min
+             c = t.values.map(&:keys)
+             cm = c.map(&:min).min
+             cx = c.map(&:max).max
+             rm && rx && (rm..rx).map{|r|
+               {_: :tr, c: 
+                 t[r].do{|r|
+                   (cm..cx).map{|c|
+                     r[c].do{|c|
+                       {_: :td, class: :cell, c:(Fn 'view/'+(a||e.q['cellview']||'title'),c,e)}
+                     }||{_: :td}}}}} || ''
+           }}}]
+    else
+      "table= layout arg required"
+    end}
+
+  fn 'view/table',->i,e{[H.css('/css/table'),(Fn 'table',i.values,e)]}
+
+  fn 'table',->es,q=nil{ p = {}
+    es.map{|e|e.respond_to?(:keys) &&
+              e.keys.map{|k|p[k]=true}}
+    keys = p.keys
+    keys.empty? ? es.html :
+    H({_: :table,:class => :tab,
+        c: [{_: :tr, c: keys.map{|k|{_: :th, class: :label, property: k, c: k.abbrURI}}},
+            *es.map{|e|
+              {_: :tr, about: e.uri, c: keys.map{|k| {_: :td, property: k, c: k=='uri' ? e.E.html : e[k].html}}}}]})}
+
+end