infod 0.0.3.1 → 0.0.3.2

data/infod/edit.rb

@@ -1,73 +1,89 @@
-#watch __FILE__
+watch __FILE__
 class E
 
   Prototypes = {
     SIOCt+'MicroblogPost' => [Content],
-    SIOCt+'BlogPost' => [Date, Title, Content], nil=>nil,
-    'blank'=>[]
+    SIOCt+'BlogPost' => [Title, Content],
+    SIOCt+'WikiArticle' => [Title, Content],
   }
 
-  F['protograph/editable'] = F['protograph/_']
+  # 404 -> create resource
+  F['protograph/create'] = -> e,env,g {
+    env['view'] = 'create'
+    F['protograph/blank'][e,env,g]}
 
-  fn 'graph/editable',->e,env,g{
-    e.fromStream g, :triplrFsStore}
-
-  # select a prototype graph
-  # , or go blank
+  # prototype select
   fn 'view/create',->g,e{
-    [H.css('/css/create'),{_: :b, c: :create},
-     Prototypes.map{|s,_| s.nil? ? {_: :b, c: ' '} : {_: :a, href:  e['REQUEST_PATH']+'?graph=edit&prototype='+(CGI.escape s), c: s.label}}]}
+    [{_: :style, c: 'a {display:block;font-size:2em}'},{_: :b, c: :type},
+     Prototypes.map{|t,ps|
+       {_: :a, href:  e['REQUEST_PATH']+'?graph=edit&prototype='+(CGI.escape t), c: t.label}}]}
 
+  # editable triples
+  F['protograph/edit'] = -> e,env,g {
+    env['view'] ||= 'edit'          # use edit-view
+    g[e.uri+'#'] = {}               # add current resource
+    rand.to_s.h}
+    
+  fn 'graph/edit',->e,env,g{
+    e.fromStream g, :triplrDoc} # add fs-sourced triples
+    
+=begin HTML <form> RDF editor
+      arg
+      prototype - initialize fields for a resource-type
+      predicate - initialize field for a particular predicate
+=end
   fn 'view/edit',->g,e{
+
+    # render a triple
     triple = ->s,p,o{
       if s && p && o
         s = s.E
         p = p.E
-        oE = p.literal o
-        (id = s.concatURI(p).concatURI oE
-        [(case p
+        oE = p.literal o                 # cast literal to URI
+        id = s.concatURI(p).concatURI oE # triple identifier
+        [(case p.uri                     # more to support here.. http://dev.w3.org/html5/markup/input.html#input
           when Content
-            {_: :textarea, name: id, c: o, rows: 24, cols: 80}
+            [{_: :textarea, name: id, c: o, rows: 16, cols: 80},
+            '<br>',o]
           when Date
             {_: :input, name: id, type: :datetime, value: o.empty? ? Time.now.iso8601 : o}
           else
             {_: :input, name: id, value: o, size: 54}
           end
-          ),"<br>\n"]) if oE
+          ),"<br>\n"]
       end}
-    
-    ps = []
-    e.q['prototype'].do{|pr| Prototypes[pr].do{|v|
-        g[e['uri']+'#'] ||= {}
-        ps.concat v }}
-    e.q['p'].do{|p| ps.push p }
+   
+    ps = [] # predicates to go editable on
+    e.q['prototype'].do{|pr|
+      Prototypes[pr].do{|v|ps.concat v }} # prototype imports
+    e.q['predicate'].do{|p|ps.push p }    # explicit predicate
 
-    [(H.once e, 'edit', (H.css '/css/edit')),
+    [{_: :style, c: ".abbr {display: none}\ntd {vertical-align:top}"},
      {_: :form, name: :editor, method: :POST, action: e['REQUEST_PATH'],
 
-       # each resource
-       c: [ g.map{|s,r|
-              url = s.E.localURL e 
-              # per-resource links
-              {class: :resource, c:
-              [{_: :a, class: :uri, id: s, c: s, href: url},
-               {_: :a, class: :edit, c: '+predicate', href: url+'?graph=_&view=addP'},'<br><br>',
-
-               # each property
-#               r.keys.concat(ps).uniq.-(['uri']).map{|p|
-               (r.keys.concat(ps).uniq.map{|p|
-                 [{_: :b, c: p}, '<br>',
-                  r[p].do{|o| [*o].map{|o|triple[s,p,o]}}, # existing triples
-                  triple[e['uri'],p,''], '<br>']} if r.class==Hash)]} if s.match(/#/)}, # create triple
-       {_: :input, type: :submit, value: 'save'}]}]}
+       c: [{_: :a, class: :edit, c: '+add field',
+             href: e['uri']+'?graph=blank&view=addP', style: 'background-color:#0f0;border-radius:5em;color:#000;padding:.5em'},
+           g.map{|s,r| # subject
+             uri = s.E.localURL e
+             {_: :table, style: 'background-color:#eee',
+               c: [{_: :tr, c: {_: :td, colspan: 2, c: {_: :a, class: :uri, id: s, c: s, href: uri}}},
+                   {_: :input, type: :hidden, name: s.E.concatURI(Edit.E).concatURI(E['/']), value: e['uri']+'?graph=edit'},
+                   r.keys.-([Edit]).concat(ps).uniq.map{|p| # resource + prototype/initialize predicates
+                     {_: :tr,
+                       c: [{_: :td, c: {_: :a, title: p, href: p, c: p.abbrURI}}, # property
+                           {_: :td,
+                             c: [r[p].do{|o|                            # objects
+                                   (o.class == Array ? o : [o]).map{|o| # each object
+                                     triple[s,p,o]}},                   # existing triples
+                                 triple[s,p,'']]}]}}]}},                # new triple
+           {_: :input, type: :submit, value: 'save'}]}]}
 
   # select a property to edit
   fn 'view/addP',->g,e{
-    [(H.once e, 'edit', (H.css '/css/edit')),
-     [Date,Title,Creator,Content,Label].map{|p|{_: :a, href: p, c: p.label+' '}},
-
+    [[Date,Title,Creator,Content,Label].map{|p|[{_: :a, href: p, c: p},'<br>']},
      {_: :form, action: e['REQUEST_PATH'], method: :GET,
-       c: [{_: :input, type: :url, name: :p, pattern: '^http.*$', size: 64},
+       c: [{_: :input, type: :url, name: :predicate, pattern: '^http.*$', size: 64},
+           {_: :input, type: :hidden, name: :graph, value: :edit},
            {_: :input, type: :submit, value: 'property'}]}]}
 
 end

data/infod/feed.rb

@@ -14,33 +14,45 @@ module FeedParse
                    end) end
 
   def parse
-
-    #prefix table
-    x={}
+    x={} # populate XMLns prefix table
     match(/<(rdf|rss|feed)([^>]+)/i)[2].scan(/xmlns:?([a-z]+)?=["']?([^'">\s]+)/){|m|x[m[0]]=m[1]}
 
-    #items
-    scan(%r{<(rss:|atom:)?(item|entry)([\s][^>]*)?>(.*?)</\1?\2>}mi){|m|
-
-      # identifier select -> RDF URI || <id> || <link>
-      u = m[2] && (u = m[2].match /about=["']?([^'">\s]+)/) && u[1] ||
-          m[3] && (((u = m[3].match /<(gu)?id[^>]*>([^<]+)/) || (u = m[3].match /<(link)>([^<]+)/)) && u[2])
-
-      yield u, E::Type, (E::SIOCt+'BlogPost').E
-      yield u, E::Type, (E::SIOC+'Post').E
-
-      #links
-      m[3].scan(%r{<(link|enclosure|media)([^>]+)>}mi){|e|
-        e[1].match(/(href|url|src)=['"]?([^'">\s]+)/).do{|url|
-          yield(u,E::Atom+'/link/'+((r=e[1].match(/rel=['"]?([^'">\s]+)/)) ? r[1] : e[0]), url[2].E)}}
-
-      #elements
-      m[3].scan(%r{<([a-z]+:)?([a-z]+)([\s][^>]*)?>(.*?)</\1?\2>}mi){|e|
-        yield u,                           # s
-        (x[e[0]&&e[0].chop]||E::RSS)+e[1], # p
-        e[3].extend(FeedParse).guess.do{|o|# o
-          o.match(/\A(\/|http)[\S]+\Z/) ? o.E : o
-        }}}
+    # scan for resources
+    scan(%r{<(?<ns>rss:|atom:)?(?<tag>item|entry)(?<attrs>[\s][^>]*)?>(?<inner>.*?)</\k<ns>?\k<tag>>}mi){|m|
+      # identifier search
+      attrs = m[2]
+      inner = m[3]
+      u = attrs.do{|a| # RDF-style identifier (RSS 1.0)
+        a.match(/about=["']?([^'">\s]+)/).do{|s|
+          s[1] }} ||
+      (inner.match(/<link>([^<]+)/) || # <link> child-node or href attribute
+       inner.match(/<link[^>]+rel=["']?alternate["']?[^>]+href=["']?([^'">\s]+)/) ||
+       inner.match(/<(?:gu)?id[^>]*>([^<]+)/)).do{|s| s[1]} # <id> child 
+
+      if u
+        if !u.match /^http/
+          puts "no HTTP URIs found #{u}"
+          u = '/junk/'+u.gsub('/','.')
+        end
+        yield u, E::Type, (E::SIOCt+'BlogPost').E
+        yield u, E::Type, (E::SIOC+'Post').E
+        
+        #links
+        inner.scan(%r{<(link|enclosure|media)([^>]+)>}mi){|e|
+          e[1].match(/(href|url|src)=['"]?([^'">\s]+)/).do{|url|
+            yield(u,E::Atom+'/link/'+((r=e[1].match(/rel=['"]?([^'">\s]+)/)) ? r[1] : e[0]), url[2].E)}}
+        
+        #elements
+        inner.scan(%r{<([a-z]+:)?([a-z]+)([\s][^>]*)?>(.*?)</\1?\2>}mi){|e|
+          yield u,                           # s
+          (x[e[0]&&e[0].chop]||E::RSS)+e[1], # p
+          e[3].extend(FeedParse).guess.do{|o|# o
+            o.match(/\A(\/|http)[\S]+\Z/) ? o.E : E::F['cleanHTML'][o]
+          }}
+      else
+        puts "no post-identifiers found #{u}"
+      end
+      }
     
     nil
   end
@@ -56,8 +68,24 @@ class E
   def listFeeds; (nokogiri.css 'link[rel=alternate]').map{|u|E (URI uri).merge(u.attr :href)} end
   alias_method :feeds, :listFeeds
 
-  def getFeed       g; insertDocs :triplrFeed, g end
-  def getFeedReddit g; insertDocs :triplrFeedReddit, g end
+  # add existing resources to index
+  #
+  # 'http:/'.E.take.select{|e|e.ext=='e'}.map{|r|E::FeedArchiver[r,r.graph,'localhost']}
+
+  FeedArchiver = -> doc, graph, host {
+    doc.roonga host
+    graph.map{|u,r|
+      r[Date].do{|t| # link doc to date-index
+        t = t[0].gsub(/[-T]/,'/').sub /(.00.00|Z)$/, '' # trim normalized timezones and non-unique symbols
+        stop = /\b(at|blog|com(ments)?|html|info|org|photo|p|post|r|status|tag|twitter|wordpress|www|1999|2005)\b/
+        b = (u.sub(/http:\/\//,'.').gsub(/\W/,'..').gsub(stop,'').sub(/\d{12,}/,'')+'.').gsub /\.+/,'.'
+        doc.ln E["http://#{host}/news/#{t}#{b}e"]}}
+  doc}
+
+  GREP_DIRS.push /^\/news\/\d{4}/
+
+  def getFeed       g; addDocs :triplrFeed, g, nil, FeedArchiver end
+  def getFeedReddit g; addDocs :triplrFeedReddit, g, nil, FeedArchiver end
 
   def triplrFeed &f 
     dateNorm :contentURIresolve,:triplrFeedNormalize,:triplrFeedRaw,&f

data/infod/fs.rb

@@ -50,10 +50,12 @@ class E
   end
   alias_method :m, :mtime
 
-  def triplrInode children=true
-    e && (d? && (yield uri, Posix + 'dir#parent', parent
-                 children && c.map{|c| yield uri, Posix + 'dir#child', c})
-          node.stat.do{|s|[:size,:ftype,:mtime].map{|p| yield uri, Stat+p.to_s, (s.send p)}})
+  def triplrInode
+    if d?
+      yield uri, Posix+'dir#parent', parent
+      c.map{|c| yield uri, Posix + 'dir#child', E[c.uri.gsub('?','%3F').gsub('#','23')]}
+    end
+    node.stat.do{|s|[:size,:ftype,:mtime].map{|p| yield uri, Stat+p.to_s, (s.send p)}}
   end
   
   def triplrSymlink

data/infod/glob.rb

@@ -1,7 +1,6 @@
 #watch __FILE__
 class E
 
-  # glob :: pattern -> [E]
   def glob p=""
     (Pathname.glob d + p).map &:E
   end
@@ -16,10 +15,10 @@ class E
 
   def docs
     base = docBase
-    [(base if pathSegment!='/' && base.e),
-     (self if base != self && e && uri[-1]!='/'),
+    [(base if pathSegment!='/' && base.e),         # doc-base
+     (self if base != self && e && uri[-1]!='/'),  # requested path
      base.glob(".{e,html,n3,nt,owl,rdf,ttl,txt}"), # docs
-     ((d? && uri[-1]=='/' && uri.size>1) ? c : []) # trailing slash = children
+     ((d? && uri[-1]=='/' && uri.size>1) ? c : []) # trailing slash -> child resources
     ].flatten.compact
   end
 

data/infod/graph.rb

@@ -1,17 +1,15 @@
 #watch __FILE__
 class E
 =begin
-  graph resolution is two-pass
+  graph construction is two-pass:
 
-  protograph/
- the first-pass will determine if the second-pass needs to run. an eTag will be derived from the return-value and any graph additions preserved for the next pass. ideal fingerprint sources include filestats, mtime checks, extremely trivial SPARQL queries, SHA160 hashes of in-RAM entities. you can define only a second or first-pass and get default behaviour for the other. for more ideas see <http://tools.ietf.org/html/draft-ietf-httpbis-p4-conditional-25#section-2.3>
+ the first-pass will signify if the second-pass needs to be run. an eTag is be derived from the return-value, ideal fingerprint sources include filestats, mtime checks, extremely trivial SPARQL queries, SHA160 hashes of in-RAM entities.. <http://tools.ietf.org/html/draft-ietf-httpbis-p4-conditional-25#section-2.3>
 
-  graph/
-   a second-pass might query a CBD (concise-bounded description) from a SPARQL store. infod was originally developed as an alternative to fragility & latency of relying on (large, hard-to-implement, must be running, configured & connectable) SPARQL stores by using the filesystem as much as possible, to experiment with hybrids like "touch" a file on successful POSTs so a store only has to be queried occasionally, and to facilitate simply hooking up bits of Ruby code to names rather than try to shoehorn what you're trying to say into some QueryLang where you're additionally without standard library functions necessitating more roundtrips and latency via marshalling/unmarshalling, parsing, loopback network-abstraction, nascent RDF-via-SPARQL-as-ORM-libs.. but go nuts experimenting w/ graph-handlers for this stuff,,i do..
+   second-pass might fetch RDF from a SPARQL store. this lib was developed as an alternative to relying on (large, hard-to-implement, must be running, configured & connectable) SPARQL stores by using the filesystem as much as possible, to experiment with hybrids like SPARQLING up a set of files to be returned in standard Apache-as-static-fileserver fashion, and to webize all sorts of non-RDF like email, directories, plain-text etc
 
-  triple streams are functions which yield triples
-  s,p - URI in String , o - Literal or URI (object must respond to #uri such as '/path'.E or {'uri' => '/some.n3'}
-  these can be formed into pipelines. the data ingesting/massaging stream-processors in feed.rb are modularized this way
+  triple streams - a source function yields triples up to the caller as it finds them,
+  a function providing just a block (consume yielded values) is a sink, both is a filter 
+  these can be stacked into pipelines. see the data-massaging stream-processors in feed.rb
 
 =end
 
@@ -29,36 +27,32 @@ class E
  * behave as normal triplr to caller, with
    side-effect of import/indexing to knowledgebase
 =end
-  def insertDocs triplr, host, p=[], &b
+  def addDocs triplr, host, p=nil, hook=nil, &b
     graph = fromStream({},triplr)
     docs = {}
     graph.map{|u,r|
-      e = u.E                # resource
-      doc = e.ef             # doc
-      doc.e ||               # exists?
-      (docs[doc.uri] ||= {}  # init doc graph
-       docs[doc.uri][u] = r  # add to graph
-       p.map{|p|             # each indexable property
-         r[p].do{|v|         # values exists?
-           v.map{|o|         # each value
-             e.index p,o}}})}# add to property index 
-    docs.map{|doc,g|
-      d = doc.E
-      if !d.e
-        d.w g, true   # write doc
-        d.roonga host # text index
-        puts "#{doc} < #{g.keys.join ' '}"
-      end}
-    graph.triples &b if b # emit the triples
+      e = u.E                 # resource
+      doc = e.ef              # doc
+      doc.e ||                # exists - we're nondestructive here
+      (docs[doc.uri] ||= {}   # init doc-graph
+       docs[doc.uri][u] = r   # add to graph
+       p && p.map{|p|         # index predicate
+         r[p].do{|v|v.map{|o| # values exist?
+             e.index p,o}}})} # index triple
+    docs.map{|d,g|            # resources in docs
+      d = d.E; puts "+doc #{d}"
+      d.w g,true              # write
+      hook[d,g,host] if hook} # insert-hook
+    graph.triples &b if b     # emit triples
     self
   end
 
-  # default protograph - identity + resource-thunks
+  # default protograph - identity < lazy-expandable resource-thunks
   # Resource, Query, Graph -> graphID
   fn 'protograph/',->e,q,g{
      g['#'] = {'uri' => '#'}
     set = (q['set'] && F['set/'+q['set']] || F['set/'])[e,q,g]
-    if set.empty?
+    if !set || set.empty?
       g.delete '#'
     else
       g['#'][RDFs+'member'] = set
@@ -96,31 +90,19 @@ class E
     # cleanup unexpanded thunks
     m.delete_if{|u,r|r.class==E}}
 
-  fn 'filter/set',->e,m,r{
-    # filter to RDFs set-members, gone will be:
-    # data about docs containing the data
-    # other fragments in a doc not matching search when indexed per-fragment
-    f = m['#'].do{|c| c[RDFs+'member'].do{|m| m.map &:uri }} || [] # members
-    m.keys.map{|u| m.delete u unless f.member? u}} # trim
-
   def graphFromFile g={}
-    if !e
-      puts "missing file! "+d
-      return
-    end
-    _ = self
-    triplr = @r.do{|r|
-                    r.q['triplr'].do{|t|
-                          respond_to?(t) && t }} || :triplrMIME
-    unless ext=='e' # native graph-format
-      _ = E '/E/rdf/' + [triplr,uri].h.dive
-      unless _.e && _.m > m;       # up to date?
-        e = {} ; puts "< #{uri}"
-        [:triplrInode,triplr].each{|t| fromStream e, t }
-        _.w e, true
+    return unless e
+    doc = self
+    unless ext=='e' # already native-format
+      triplr = @r.do{|r|r.q['triplr'].do{|t| (respond_to? t) && t }} || :triplrMIME
+      doc = E '/E/rdf/' + [triplr,uri].h.dive
+      unless doc.e && doc.m > m; # freshness check
+        graph = {}
+        [:triplrInode,triplr].each{|t| fromStream graph, t }
+        doc.w graph, true
       end
     end
-    g.mergeGraph _.r true
+    g.mergeGraph doc.r true
   end
 
   def graph g={}
@@ -128,4 +110,38 @@ class E
     g
   end
 
+  def triplrJSON
+    yield uri, '/application/json', (JSON.parse read) if e
+  rescue Exception => e
+  end
+
+  def to_json *a
+    to_h.to_json *a
+  end
+
+  fn Render+'application/json',->d,_=nil{[d].to_json}
+
+end
+
+class Hash
+
+  def graph g
+    g.merge!({uri=>self})
+  end
+
+  def mergeGraph g
+    g.triples{|s,p,o|
+      self[s] = {'uri' => s} unless self[s].class == Hash 
+      self[s][p] ||= []
+      self[s][p].push o unless self[s][p].member? o } if g
+    self
+  end
+
+  def triples &f
+    map{|s,r|
+      r.map{|p,o|
+        o.class == Array ? o.each{|o| yield s,p,o} : yield(s,p,o) unless p=='uri'} if r.class == Hash
+    }
+  end
+
 end

data/infod/grep.rb

@@ -1,12 +1,17 @@
 #watch __FILE__
 class E
 
+  fn 'view/'+Posix+'util#grep',-> d,e {{_: :form, c: [{_: :input, name: :q, style: 'font-size:2em'},{_: :input, type: :hidden, name: :set, value: :grep}]}}
+
+  GREP_DIRS=[]
+
   fn 'set/grep',->e,q,m{
-    q['i'] ||= true # case-insensitive by default
-    q['q'].do{|query|
-      [e,e.pathSegment].compact.select(&:e).map{|e|
-        grep = "grep -rl#{q.has_key?('i') ? 'i' : ''} #{query.sh} #{e.sh}"
-        `#{grep}`}.map{|r|r.lines.to_a.map{|r|r.chomp.unpathFs}}.flatten}}
+    q['q'].do{|query| m[e.uri+'#grep'] = {Type => E[Posix+'util#grep']}
+      path = e.pathSegment
+      GREP_DIRS.find{|p|path.uri.match p}.do{|allow|
+        [e,path].compact.select(&:e).map{|e|
+          `grep -irl #{query.sh} #{e.sh} | head -n 200`}.map{|r|r.lines.to_a.map{|r|r.chomp.unpathFs}}.flatten
+      }||(puts "no grep available on #{path}")}}
 
   fn 'view/grep',->d,e{
     w = e.q['q']
@@ -24,8 +29,7 @@ class E
       # sequential pattern
       p = /#{w.join '.*'}/i
 
-      [H.css('/css/search'), H.css('/css/grep'),
-       F['view/'+Search][e.q,e],
+      [H.css('/css/grep'),
        {_: :style, c: c.values.map{|i|
            # word color
            b = rand(16777216)
@@ -54,8 +58,7 @@ class E
              # exerpt
              l[0..403].gsub(a){|g|
                H({_: :span, class: "w w#{c[g.downcase]}",c: g})}
-           },"<br>"]]},
-       {_: :a, class: :down, href: e['REQUEST_PATH']+e.q.except('view').qs, style: "background-color: #{E.cs}",c: '&darr;'}]
+           },"<br>"]]}]
     end }
 
 end