pho 0.3 → 0.4

data/CHANGES

@@ -1,5 +1,14 @@
 Brief notes on significant changes and additions in each release
 
+== Version 0.4
+
+* Added support for reading Job lifecycle data exposed in Talis Platform Release 21
+* Added constants for 2 new analysers added in Talis Platform Release 21
+* Refactored job submission code to move methods from the Store class onto the Jobs and Job classes instead.
+  This brings that code in line with the patterns used elsewhere
+* Re-implemented snapshot reading code to use RDF/XML rather than scraping the HTML  
+* Added a fledgling command-line tool for working with the platform "pho_store"
+
 == Version 0.3
 
 * Added support for QueryProfiles, Facets

data/Rakefile

@@ -5,7 +5,7 @@ require 'rake/testtask'
 require 'rake/clean'
 
 NAME = "pho"
-VER = "0.3"
+VER = "0.4"
 
 RDOC_OPTS = ['--quiet', '--title', 'Pho (Talis Platform Client) Reference', '--main', 'README']
 
@@ -31,6 +31,7 @@ SPEC =
     s.files = PKG_FILES
     s.require_path = "lib" 
     s.bindir = "bin"
+    s.executables = ["talis_store"]
     s.test_file = "tests/ts_pho.rb"
     s.add_dependency("httpclient", ">= 2.1.3.1")
     s.add_dependency("json", ">= 1.1.3")

data/bin/talis_store

@@ -0,0 +1,263 @@
+#!/usr/bin/env ruby
+require 'rubygems'
+require 'pho'
+require 'getoptlong'
+
+PROGRAM     = File::basename $0
+
+USAGE = <<-EOL
+SYNOPSIS
+
+  #{ PROGRAM } mode [args]*
+
+DESCRIPTION
+
+  Provides a simple command-line interface for working with a Talis Platform store. The script attempts to provide 
+  access to the most useful core functionality, making it easier to write scripts that interact with a store, 
+  as well as providing a simple set of developer tools. 
+    
+MODE
+
+  backup
+    Take a snapshot of a store and then download that new snapshot to the local system
+    By default the snapshot is downloaded into /tmp, but can be specified on the command-line
+     
+    example :
+       
+      #{ PROGRAM } backup -u user -p pass -s my-dev1
+      #{ PROGRAM } backup -u user -p pass -s my-dev1 -d ~/backups
+
+  describe
+    Fetch description of a resource.   
+  
+    example :
+  
+      #{ PROGRAM } describe -u user -p pass -s my-dev1 -r http://www.example.org/foo
+
+  reindex
+    Trigger a reindex job on the store
+      
+    example :
+    
+      #{ PROGRAM } reindex -u user -p pass -s my-dev1
+  
+  reset
+    Reset the store. THIS WILL DELETE ALL DATA.
+      
+    example :
+    
+      #{ PROGRAM } reset -u user -p pass -s my-dev1
+  
+  restore
+    Restore the store from a snapshot. By default it will restore from the most recent snapshot
+    held in the Platform. Specify a url parameter to indicate a url from which a snapshot can be downloaded.
+      
+    example :
+    
+      #{ PROGRAM } restore -u user -p pass -s my-dev1                              
+      #{ PROGRAM } restore -u user -p pass -s my-dev1 -r http://www.example.org/backups/snapshot.tar      
+    
+  snapshot
+    Generate a snapshot of the store. (This doesnt download the snapshot, use backup for that).
+      
+    example :
+  
+      #{ PROGRAM } snapshot -u user -p pass -s my-dev1
+
+  sparql
+    Run a SPARQL query against the Platform.
+    
+    example :
+    
+      #{ PROGRAM } sparql -f ~/queries/query.rq
+    
+  status
+    Show read-write status of the store
+    
+  store
+    Upload some RDF data to the Platform. Can either specify a local file, or a URL
+    When a url is specified, then the data is retrieved and then submitted to the Platform
+    Can also be used to store an entire directory of RDF/XML using the --dir option.
+       
+    example :
+  
+      #{ PROGRAM } store -u user -p pass -s my-dev1 -f /tmp/data.rdf
+      #{ PROGRAM } store -u user -p pass -s my-dev1 -r http://www.example.net/data/somedata.rdf
+      #{ PROGRAM } store -u user -p pass -s my-dev1 -d ~/data
+           
+OPTIONS
+
+  --help            , -h
+    show this message
+  --username        , -u
+    specify username
+  --password        , -p
+    specify password
+  --store           , -s
+    specify store name. Can be just name, or full URL
+  --url             , -r
+    specify a url, e.g. for retrieving description or uploading
+  --file            , -f
+    specify a file name, e.g. data to upload
+  --dir
+    specify a directory, e.g. containing data to upload or location to place data.
+        
+EOL
+
+mode = ARGV.shift
+
+opts = GetoptLong::new(
+         [ "--help"            , "-h" , GetoptLong::NO_ARGUMENT ],
+         [ "--username"        , "-u" , GetoptLong::REQUIRED_ARGUMENT ],
+         [ "--password"        , "-p" , GetoptLong::REQUIRED_ARGUMENT ],
+         [ "--store"           , "-s" , GetoptLong::REQUIRED_ARGUMENT ],
+         [ "--url"             , "-r" , GetoptLong::REQUIRED_ARGUMENT ],
+         [ "--file"            , "-f" , GetoptLong::REQUIRED_ARGUMENT ],
+         [ "--dir"             , "-d" , GetoptLong::REQUIRED_ARGUMENT ]                           
+       ).enum_for.inject({}) { |h, (k, v)| h.update k.delete('-') => v }
+
+mode = "help" if opts["--help"]
+
+if mode == "help"
+  USAGE.display  
+else
+
+  if !opts["username"]
+    puts "no <username>"
+    abort USAGE
+  end
+  if !opts["password"]
+    puts "no <password>"
+    abort USAGE
+  end  
+  if !opts["store"]
+    puts "no <store>"
+    abort USAGE
+  end  
+  
+  storename = opts["store"]
+  if !storename.start_with?("http")
+      storename = "http://api.talis.com/stores/#{storename}"  
+  end
+  
+  store = Pho::Store.new(storename, opts["username"], opts["password"])
+  
+  #begin
+    
+    case mode
+      when "status"
+         status = Pho::Status.read_from_store(store)
+         puts "Store Status:\nReadable: #{status.readable?}\nWritable: #{status.writeable?}" 
+      when "backup"
+         puts "Submitting Snapshot Job"
+         resp = Pho::Jobs.submit_snapshot(store, "Reindex", Time.now)
+         puts "Monitoring Job"
+         job = Pho::Jobs.wait_for_submitted(resp, store) do |job, message, time|
+           puts "#{time} #{message}"
+         end 
+         puts "Job Completed"         
+         snapshot = Pho::Snapshot.read_from_store(store)
+         puts "Retrieving #{snapshot.url}"
+         dir = opts["dir"] || Dir.tmpdir
+         snapshot.backup(store, dir)
+         puts "Download complete. MD5 OK."
+      when "snapshot"
+         puts "Submitting Snapshot Job"
+         resp = Pho::Jobs.submit_snapshot(store, "Reindex", Time.now)
+         puts "Monitoring Job"
+         job = Pho::Jobs.wait_for_submitted(resp, store) do |job, message, time|
+           puts "#{time} #{message}"
+         end 
+        puts "Snapshot Completed"                   
+      when "reindex"
+         puts "Submitting Reindex Job"
+         resp = Pho::Jobs.submit_reindex(store, "Reindex", Time.now)
+         if resp.status == 201
+           puts "Monitoring Reindex Job: #{resp.header["Location"].first}"
+         end
+         job = Pho::Jobs.wait_for_submitted(resp, store) do |job, message, time|
+           puts "#{time} #{message}"
+         end 
+         puts "Reindex Completed"
+      when "reset"
+        puts "Resetting Store"
+        resp = Pho::Jobs.submit_reset(store, "Reset", Time.now)
+        puts "Monitoring Reset Job:"
+        job = Pho::Jobs.wait_for_submitted(resp, store) do |job, message, time|
+          puts "#{time} #{message}"
+        end 
+        puts "Reset Completed"
+      when "restore"
+        url = opts["url"]     
+        if url == nil
+          puts "Restore from latest snapshot"
+          snapshot = Pho::Snapshot.read_from_store(store)
+          url = snapshot.url
+        end
+        puts "Restoring from #{url}"    
+        resp = Pho::Jobs.submit_restore(store, url, "Reset", Time.now)
+        job = Pho::Jobs.wait_for_submitted(resp, store) do |job, message, time|
+          puts "#{time} #{message}"
+        end 
+        puts "Restore Completed"
+      when "store"
+        resp = nil
+        if opts["url"]
+          puts "Storing remote data: #{opts["url"]}"
+          resp = store.store_url( opts["url"] ) 
+        elsif opts["file"]
+          puts "Storing local file: #{opts["file"]}"
+          resp = store.store_file( File.new( opts["file"] ) )
+        elsif opts["dir"]
+          puts "Storing contents of directory: #{opts["dir"]}"
+          collection = Pho::RDFCollection.new(store, opts["dir"])         
+          collection.store()          
+          puts collection.summary()
+        else     
+          abort USAGE
+        end 
+      
+        if resp != nil 
+          if resp.status == 204
+            puts "Complete"
+          else
+            puts "Error: #{resp.status} #{resp.reason}"
+            puts resp.content
+          end         
+        end
+      
+      when "describe"
+        if opts["url"] == nil
+          abort USAGE
+        end 
+        resp = store.describe( opts["url"] )
+        if resp.status == 200
+          puts resp.content
+        else
+          puts "Error: #{resp.status} #{resp.reason}"
+          puts resp.content
+        end
+      when "sparql"
+        if opts["file"] == nil
+          abort USAGE
+        end 
+        query = File.new( opts["file"] ).read()
+        resp = store.sparql(query)
+        if resp.status == 200
+          puts resp.content
+        else
+          puts "Error: #{resp.status} #{resp.reason}"
+          puts resp.content          
+        end
+                            
+      else
+         abort USAGE
+    end  
+   
+  #rescue StandardError => e
+  #   puts "Failed!"
+  #   puts e
+  #end
+      
+end  
+ 

data/doc/rdoc/classes/Pho.html

@@ -124,13 +124,15 @@
       <h3 class="section-bar">Classes and Modules</h3>
 
       Module <a href="Pho/Facet.html" class="link">Pho::Facet</a><br />
-Module <a href="Pho/Namespaces.html" class="link">Pho::Namespaces</a><br />
 Class <a href="Pho/Analyzers.html" class="link">Pho::Analyzers</a><br />
 Class <a href="Pho/DatatypeProperty.html" class="link">Pho::DatatypeProperty</a><br />
 Class <a href="Pho/Etags.html" class="link">Pho::Etags</a><br />
 Class <a href="Pho/FieldPredicateMap.html" class="link">Pho::FieldPredicateMap</a><br />
 Class <a href="Pho/FieldWeighting.html" class="link">Pho::FieldWeighting</a><br />
 Class <a href="Pho/Job.html" class="link">Pho::Job</a><br />
+Class <a href="Pho/JobUpdate.html" class="link">Pho::JobUpdate</a><br />
+Class <a href="Pho/Jobs.html" class="link">Pho::Jobs</a><br />
+Class <a href="Pho/Namespaces.html" class="link">Pho::Namespaces</a><br />
 Class <a href="Pho/QueryProfile.html" class="link">Pho::QueryProfile</a><br />
 Class <a href="Pho/RDFCollection.html" class="link">Pho::RDFCollection</a><br />
 Class <a href="Pho/Snapshot.html" class="link">Pho::Snapshot</a><br />
@@ -159,26 +161,6 @@ Class <a href="Pho/Store.html" class="link">Pho::Store</a><br />
           <td>=</td>
           <td class="context-item-value">{&quot;Content-Type&quot;=&gt;&quot;application/rdf+xml&quot;}.freeze</td>
         </tr>
-        <tr class="top-aligned-row context-row">
-          <td class="context-item-name">JOB_RESET</td>
-          <td>=</td>
-          <td class="context-item-value">&quot;http://schemas.talis.com/2006/bigfoot/configuration#ResetDataJob&quot;.freeze</td>
-        </tr>
-        <tr class="top-aligned-row context-row">
-          <td class="context-item-name">JOB_SNAPSHOT</td>
-          <td>=</td>
-          <td class="context-item-value">&quot;http://schemas.talis.com/2006/bigfoot/configuration#SnapshotJob&quot;.freeze</td>
-        </tr>
-        <tr class="top-aligned-row context-row">
-          <td class="context-item-name">JOB_REINDEX</td>
-          <td>=</td>
-          <td class="context-item-value">&quot;http://schemas.talis.com/2006/bigfoot/configuration#ReindexJob&quot;.freeze</td>
-        </tr>
-        <tr class="top-aligned-row context-row">
-          <td class="context-item-name">JOB_RESTORE</td>
-          <td>=</td>
-          <td class="context-item-value">&quot;http://schemas.talis.com/2006/bigfoot/configuration#RestoreJob&quot;.freeze</td>
-        </tr>
         <tr class="top-aligned-row context-row">
           <td class="context-item-name">READ_ONLY</td>
           <td>=</td>

data/doc/rdoc/classes/Pho/Analyzers.html

@@ -263,6 +263,28 @@ their unaccented equivalent See API documentation at <a
 href="http://n2.talis.com/wiki/Field_Predicate_Map">n2.talis.com/wiki/Field_Predicate_Map</a>
 for details of replacements
 
+</td>
+        </tr>
+        <tr class="top-aligned-row context-row">
+          <td class="context-item-name">PORTER_NORMALIZE_STANDARD</td>
+          <td>=</td>
+          <td class="context-item-value">&quot;http://schemas.talis.com/2007/bigfoot/analyzers#porter-norm-en&quot;.freeze</td>
+          <td width="3em">&nbsp;</td>
+          <td class="context-item-desc">
+English analyzer with porter stemming, case normalization, latin 1
+normalization, and stop words removal
+
+</td>
+        </tr>
+        <tr class="top-aligned-row context-row">
+          <td class="context-item-name">PORTER_NO_STOP_WORD_STANDARD</td>
+          <td>=</td>
+          <td class="context-item-value">&quot;http://schemas.talis.com/2007/bigfoot/analyzers#porter-nostop-norm-en&quot;.freeze</td>
+          <td width="3em">&nbsp;</td>
+          <td class="context-item-desc">
+English analyzer with porter stemming, case normalization and latin 1
+normalization.
+
 </td>
         </tr>
         </table>

data/doc/rdoc/classes/Pho/DatatypeProperty.html

@@ -92,8 +92,8 @@ Captures information about a mapped datatype from a Field Predicate Map
       <h3 class="section-bar">Methods</h3>
 
       <div class="name-list">
-      <a href="#M000034">new</a>&nbsp;&nbsp;
-      <a href="#M000035">to_rdf</a>&nbsp;&nbsp;
+      <a href="#M000044">new</a>&nbsp;&nbsp;
+      <a href="#M000045">to_rdf</a>&nbsp;&nbsp;
       </div>
     </div>
 
@@ -155,28 +155,39 @@ URI for this mapping
     <div id="methods">
       <h3 class="section-bar">Public Class methods</h3>
 
-      <div id="method-M000034" class="method-detail">
-        <a name="M000034"></a>
+      <div id="method-M000044" class="method-detail">
+        <a name="M000044"></a>
 
         <div class="method-heading">
-          <a href="DatatypeProperty.src/M000034.html" target="Code" class="method-signature"
-            onclick="popupCode('DatatypeProperty.src/M000034.html');return false;">
+          <a href="#M000044" class="method-signature">
           <span class="method-name">new</span><span class="method-args">(uri, property_uri, name, analyzer=nil)</span>
           </a>
         </div>
       
         <div class="method-description">
+          <p><a class="source-toggle" href="#"
+            onclick="toggleCode('M000044-source');return false;">[Source]</a></p>
+          <div class="method-source-code" id="M000044-source">
+<pre>
+<span class="ruby-comment cmt"># File lib/pho/field_predicate_map.rb, line 64</span>
+    <span class="ruby-keyword kw">def</span> <span class="ruby-identifier">initialize</span>(<span class="ruby-identifier">uri</span>, <span class="ruby-identifier">property_uri</span>, <span class="ruby-identifier">name</span>, <span class="ruby-identifier">analyzer</span>=<span class="ruby-keyword kw">nil</span>)
+        <span class="ruby-ivar">@uri</span> = <span class="ruby-identifier">uri</span>
+        <span class="ruby-ivar">@property_uri</span> = <span class="ruby-identifier">property_uri</span>
+        <span class="ruby-ivar">@name</span> = <span class="ruby-identifier">name</span>        
+        <span class="ruby-ivar">@analyzer</span> = <span class="ruby-identifier">analyzer</span>
+    <span class="ruby-keyword kw">end</span>
+</pre>
+          </div>
         </div>
       </div>
 
       <h3 class="section-bar">Public Instance methods</h3>
 
-      <div id="method-M000035" class="method-detail">
-        <a name="M000035"></a>
+      <div id="method-M000045" class="method-detail">
+        <a name="M000045"></a>
 
         <div class="method-heading">
-          <a href="DatatypeProperty.src/M000035.html" target="Code" class="method-signature"
-            onclick="popupCode('DatatypeProperty.src/M000035.html');return false;">
+          <a href="#M000045" class="method-signature">
           <span class="method-name">to_rdf</span><span class="method-args">(ns=true)</span>
           </a>
         </div>
@@ -191,6 +202,27 @@ rdf:Description, optionally including namespaces
 
 </td></tr>
 </table>
+          <p><a class="source-toggle" href="#"
+            onclick="toggleCode('M000045-source');return false;">[Source]</a></p>
+          <div class="method-source-code" id="M000045-source">
+<pre>
+<span class="ruby-comment cmt"># File lib/pho/field_predicate_map.rb, line 73</span>
+    <span class="ruby-keyword kw">def</span> <span class="ruby-identifier">to_rdf</span>(<span class="ruby-identifier">ns</span>=<span class="ruby-keyword kw">true</span>)
+      <span class="ruby-identifier">rdf</span> = <span class="ruby-value str">&quot;&lt;rdf:Description &quot;</span> 
+      <span class="ruby-keyword kw">if</span> <span class="ruby-identifier">ns</span>
+        <span class="ruby-identifier">rdf</span> <span class="ruby-operator">&lt;&lt;</span> <span class="ruby-node">&quot; xmlns:rdf=\&quot;http://www.w3.org/1999/02/22-rdf-syntax-ns#\&quot; xmlns:frm=\&quot;#{Pho::Namespaces::FRAME}\&quot; xmlns:bf=\&quot;#{Pho::Namespaces::CONFIG}\&quot; &quot;</span>        
+      <span class="ruby-keyword kw">end</span> 
+      <span class="ruby-identifier">rdf</span> <span class="ruby-operator">&lt;&lt;</span> <span class="ruby-node">&quot; rdf:about=\&quot;#{@uri}\&quot;&gt;&quot;</span>
+      <span class="ruby-identifier">rdf</span> <span class="ruby-operator">&lt;&lt;</span> <span class="ruby-node">&quot; &lt;frm:property rdf:resource=\&quot;#{@property_uri}\&quot;/&gt;&quot;</span>
+      <span class="ruby-identifier">rdf</span> <span class="ruby-operator">&lt;&lt;</span> <span class="ruby-node">&quot; &lt;frm:name&gt;#{@name}&lt;/frm:name&gt;&quot;</span>
+      <span class="ruby-keyword kw">if</span> <span class="ruby-ivar">@analyzer</span> <span class="ruby-operator">!=</span> <span class="ruby-keyword kw">nil</span>
+        <span class="ruby-identifier">rdf</span> <span class="ruby-operator">&lt;&lt;</span> <span class="ruby-node">&quot; &lt;bf:analyzer rdf:resource=\&quot;#{@analyzer}\&quot; /&gt;&quot;</span>  
+      <span class="ruby-keyword kw">end</span>      
+      <span class="ruby-identifier">rdf</span> <span class="ruby-operator">&lt;&lt;</span> <span class="ruby-value str">&quot;&lt;/rdf:Description&gt;&quot;</span>
+      <span class="ruby-keyword kw">return</span> <span class="ruby-identifier">rdf</span>  
+    <span class="ruby-keyword kw">end</span>
+</pre>
+          </div>
         </div>
       </div>