data_doc 0.0.2 → 0.0.3

data/History.txt

@@ -1,6 +1,14 @@
+=== 0.0.3 2012-10-22
+
+* 4 minor enhancements:
+  * RDoc coverage established
+  * Layouts
+  * Headers
+  * ERB processing under $SAFE = 4 
+  
 === 0.0.2 2012-10-18
 
-* 1 minor enhancement:
+* 2 minor enhancements:
   * Documentation fix
   * Test release
 

data/Manifest.txt

@@ -1,14 +1,19 @@
 History.txt
 Manifest.txt
-PostInstall.txt
 README.rdoc
 Rakefile
 bin/data_doc
 lib/data_doc.rb
 lib/data_doc/cli.rb
+lib/data_doc/document.rb
+lib/data_doc/store.rb
+lib/data_doc/present.rb
 script/console
 script/destroy
 script/generate
 test/test_data_doc.rb
+test/test_data_doc_document.rb
+test/test_data_doc_store.rb
+test/test_data_doc_present.rb
 test/test_data_doc_cli.rb
 test/test_helper.rb

data/README.rdoc

@@ -4,6 +4,8 @@
 
 {<img src="https://secure.travis-ci.org/alilee/data_doc.png" alt="Build Status" />}[http://travis-ci.org/alilee/data_doc]
 
+NOTE: THIS GEM DOES NOTHING YET.
+
 == DESCRIPTION:
 
 Complex documents can be challenging to author, and time-consuming to achieve 
@@ -11,6 +13,8 @@ high levels of internal consistency. This gem offers a Domain Specific Language
 to embed structured data in a markdown document and then render it into 
 configurable tables. 
 
+Note: See security implications below.
+
 == FEATURES/PROBLEMS:
 
 * Main document is Markdown for convenient content authoring.
@@ -19,25 +23,71 @@ configurable tables.
 * Generates HTML from Markdown.
 * Provides fine-grained control over the HTML headers, including 
   CSS, for the output.
-* Simple DSL for capturing structured data
+* Simple DSL for capturing structured data.
 * Simple DSL for querying data stores and laying out in tables.
+* Command line interface
+* Simple web-page interface using sinatra
+
+== SECURITY IMPLICATIONS:
+
+This gem uses ERB to process arbitrary ruby code provided by a user. 
+Some sandboxing ($SAFE) is used, but you should assess and manage 
+your level of risk.
 
 == SYNOPSIS:
 
-  data_doc your_file.md.erb > result.html
+  $ bin/data_doc --help
+  Processes structured data embedded in a markdown document and then renders it into configurable tables.
+
+  Usage: data_doc [options] filename
+
+  Options are:
+
+  Specific options:
+      -c, --connection FILENAME        Override document connection settings with FILENAME
+      -r, --read-only                  Use data already in database rather than document data
+      -d, --data-only                  Use document data but do not change database schema
+      -o, --output FILENAME            Put generated output in FILENAME
+      -f, --format TYPE                Select type of output from html (default: html)
+      -v, --[no-]verbose               Run verbosely
+
+  Common options:
+      -h, --help                       Show this message
+          --version                    Show version
+  $
   
 The data definition language looks like this:
 
+  connection adapter: sqlite3 database: '/tmp/example.db'
+  
+  store 'priority' do
+    string 'name'
+    alternate_key 'name'
+  end
+  
+  priority name: 'high'
+  priority name: 'medium'
+  priority name: 'low'
+
   store 'requirement' do
     string 'name'
     text 'description'
-    string 'priority'
+    ref 'priority', in: 'priority'
     alternate_key 'name'    
   end
   
-  requirement name: "It MUST "
+  requirement name: "It MUST be fast", description: "It must return within 5s.", priority: 'high'
+  requirement name: "It MUST look good", description: "It must be blue in colour.", priority: 'medium'
+  
+The table presentation language looks like this:
+
+  present 'select * from priority' #=> '<table>...</table>'
   
-The 
+  present requirement.project('priority', 'name') do
+    caption 'Table of requirements by priority'
+    column_order 'name', 'priority'
+    label 'priority, 'Importance'
+  end                              #=> '<table>...</table>'
 
 == REQUIREMENTS:
 

data/Rakefile

@@ -5,22 +5,14 @@ require 'fileutils'
 require './lib/data_doc'
 
 Hoe.plugin :newgem
-# Hoe.plugin :website
-# Hoe.plugin :cucumberfeatures
 
 # Generate all the Rake tasks
 # Run 'rake -T' to see list of generated tasks (from gem root directory)
 $hoe = Hoe.spec 'data_doc' do
-  self.developer 'Alister Lee', 'gems@shortepic.com'
-  self.post_install_message = 'PostInstall.txt' # TODO remove if post-install message not required
-  self.rubyforge_name       = 'data-doc' # TODO this is default value
-  # self.extra_deps         = [['activesupport','>= 2.0.2']]
-
+  self.developer        'Alister Lee', 'gems@shortepic.com'
+  self.rubyforge_name = 'data-doc'
+  self.extra_deps =     [['activerecord','>= 2.0.2']]
 end
 
 require 'newgem/tasks'
 Dir['tasks/**/*.rake'].each { |t| load t }
-
-# TODO - want other tests/tasks run by default? Add them to the list
-# remove_task :default
-# task :default => [:spec, :features]

data/lib/data_doc.rb

@@ -1,6 +1,16 @@
 $:.unshift(File.dirname(__FILE__)) unless
   $:.include?(File.dirname(__FILE__)) || $:.include?(File.expand_path(File.dirname(__FILE__)))
 
+#
+# Encompasses the data_doc classes.
+#
 module DataDoc
-  VERSION = '0.0.2'
-end
+  # Gem version
+  VERSION = '0.0.3'
+  # A summary of purpose of the tool.  
+  DESCRIPTION = 'Processes structured data embedded in a markdown document and then renders it into configurable tables.'
+end
+
+require 'data_doc/document.rb'
+# require 'data_doc/store.rb'
+# require 'data_doc/table.rb'

data/lib/data_doc/cli.rb

@@ -1,42 +1,103 @@
 require 'optparse'
+require 'yaml'
 
 module DataDoc
+  
+  #
+  # Manages command-line options and arguments.
+  #
   class CLI
+            
+    #
+    # Parses the command line and calls the main object.
+    #
+    # stdout:: IO to redirect output for testing.
+    # arguments:: contents of command line
+    #
     def self.execute(stdout, arguments=[])
 
-      # NOTE: the option -p/--path= is given as an example, and should be replaced in your application.
+      doc = DataDoc::Document.new
+      doc.output = stdout
 
-      options = {
-        :path     => '~'
-      }
-      mandatory_options = %w(  )
-
-      parser = OptionParser.new do |opts|
+      OptionParser.new do |opts|
         opts.banner = <<-BANNER.gsub(/^          /,'')
-          This application is wonderful because...
+          #{DataDoc::DESCRIPTION}
 
-          Usage: #{File.basename($0)} [options]
+          Usage: #{File.basename($0)} [options] filename
 
           Options are:
         BANNER
+        
         opts.separator ""
-        opts.on("-p", "--path PATH", String,
-                "This is a sample message.",
-                "For multiple lines, add more strings.",
-                "Default: ~") { |arg| options[:path] = arg }
-        opts.on("-h", "--help",
-                "Show this help message.") { stdout.puts opts; exit }
-        opts.parse!(arguments)
+        opts.separator "Specific options:"
 
-        if mandatory_options && mandatory_options.find { |option| options[option.to_sym].nil? }
-          stdout.puts opts; exit
+        opts.on("-c", "--connection FILENAME", 
+                "Override document connection settings with FILENAME") do |filename|
+          begin
+            doc.connection = YAML.load(File.read(filename))
+          rescue Exception => e
+            stdout.puts "ERROR with connection file (#{e.message})"
+            return 1
+          end
+        end
+        
+        opts.on("-r", "--read-only", "Use data already in database rather than document data") do |r|
+          doc.read_only = r
+        end
+        
+        opts.on("-d", "--data-only", "Use document data but do not change database schema") do |d|
+          doc.data_only = d
+        end
+
+        opts.on("-o", "--output FILENAME", 
+                "Put generated output in FILENAME") do |filename|
+          begin
+            doc.output = File.open(filename, 'w+')
+          rescue Exception => e
+            stdout.puts "ERROR with output file (#{e.message})"
+            return 1
+          end
+        end
+                
+        type_list = DataDoc::Document::OUTPUT_TYPES.join(', ')
+        opts.on("-f", "--format TYPE", DataDoc::Document::OUTPUT_TYPES, "Select type of output from #{type_list} (default: #{doc.format})") do |format|
+          doc.format = format
+        end
+
+        opts.on("-v", "--[no-]verbose", "Run verbosely") do |v|
+          doc.verbose = v
         end
-      end
 
-      path = options[:path]
+        opts.separator ""
+        opts.separator "Common options:"
 
-      # do stuff
-      stdout.puts "To update this executable, look in lib/data_doc/cli.rb"
+        opts.on_tail("-h", "--help", "Show this message") do
+          stdout.puts opts
+          return 0
+        end
+
+        opts.on_tail("--version", "Show version") do
+          stdout.puts DataDoc::VERSION
+          return 0
+        end        
+        
+        opts.parse!(arguments)
+        
+        if arguments.length != 1 
+          stdout.puts "ERROR missing input file"
+          return 1
+        end
+                
+      end
+
+      begin
+        content = File.open(arguments.first, "r")
+      rescue Exception => e
+        stdout.puts "ERROR opening content file (#{e.message})"
+        return 1
+      end
+            
+      doc.generate(content)
     end
   end
 end

data/lib/data_doc/document.rb

@@ -0,0 +1,198 @@
+require 'rdiscount'
+require 'erb'
+
+module DataDoc
+
+  #
+  # Class for processing and rendering data_docs.
+  #
+  # Manages processing and formatting of the document.
+  #
+  class Document
+              
+    # 
+    # :section: 1. Options
+    #
+    # Allows access to set processing options prior to document generation.
+    #
+    
+    #
+    # Sets up so that default options can be read.
+    #
+    def initialize
+      @format = 'html'
+      @output = STDOUT
+      @verbose = false
+      @read_only = false
+      @data_only = false
+      @connection = nil
+      @layout_filename = nil
+      
+      @headers = Array.new
+    end
+    
+    # MIME-type for output
+    attr_accessor :format 
+
+    # Available mime types that can be generated.
+    OUTPUT_TYPES = ['html']
+
+    # output filename
+    attr_accessor :output 
+
+    # display verbose output during processing
+    attr_accessor :verbose
+
+    # do not change schema or data
+    attr_accessor :read_only 
+
+    # do not change schema; truncates tables
+    attr_accessor :data_only
+    
+    # 
+    # Sets the database connection that the stores will be using
+    #
+    def connection=(connection)
+    end
+    
+    # 
+    # Sets the layout file option.
+    #
+    # See #set_layout_file below for format.
+    #
+    def layout=(filename)
+      @layout_filename = filename
+    end
+        
+    #
+    # :section: 2. Main function
+    # 
+        
+    #
+    # Generate the output for the given input content.
+    #
+    def generate(content_io)
+      erb_content = content_io.read
+      begin
+        self.untrust
+        mark_down = ERB.new(erb_content, 4).result(binding.taint) # $SAFE = 4
+      ensure
+        self.trust
+      end
+      content_html = RDiscount.new(mark_down).to_html
+      html = wrap_in_layout(content_html)
+      @output.write(html)
+      0
+    end
+    
+    #
+    # :section: 3. DSL for layout file
+    #
+    
+    #
+    # Set layout file (from the filename, not the template content itself)
+    #
+    # Content is html, with ERB placeholders to yield to add content at 
+    # appropriate points.
+    #
+    # [ <%= yield :head %> ] marks where to place any headers defined (see #meta below for an example)
+    # [ <%= yield :content %> <em>or</em> <%= yield %> ] marks where to place the processed content from the input file.
+    # 
+    def set_layout_file(filename)
+      self.layout=(filename)
+    end
+    
+    #
+    # Default layout if no layout file provided.
+    #
+    # Simplistic valid html which accepts headers and puts content directly into body tag.
+    #
+    def self.default_layout
+      "<!DOCTYPE html>\n<html>\n<head><%= yield :head %></head>\n<body>\n<%= yield %>\n</body>\n</html>"
+    end
+    
+    #
+    # :section: 4. DSL for header tags
+    #
+  
+    #
+    # Sets the title tag and emits the title in a classed div.title. 
+    #
+    def title(text)
+      add_header "<title>#{text}</title>"
+      "<div class=\"title\">#{text}</div>"
+    end
+    
+    #
+    # Adds a meta tag to the headers
+    #
+    def meta(attrs = {})
+      add_header "<meta #{html_attrs(attrs)}>"    
+    end
+
+    #
+    # Adds a script tag to the headers, yielding for body of tag.
+    #
+    def script(attrs = {})
+      add_header "<script #{html_attrs(attrs)}>#{"\n"+yield+"\n" if block_given?}</script>"    
+    end
+  
+    #
+    # Adds a link tag to the headers
+    #
+    def link(attrs = {})
+      add_header "<link #{html_attrs(attrs)}>"
+    end
+            
+    #
+    # :section: 5. Protected
+    #
+    
+  protected
+          
+    #
+    # Isolates ERB for layout
+    #
+    class IsolatedLayoutContext
+      # captures binding including any block
+      def binding_with_block
+        binding
+      end
+    end
+
+    #
+    # Wraps the default layout or a user provided layout around the content. 
+    #
+    def wrap_in_layout(content)
+      @layout ||= @layout_filename.nil? ? DataDoc::Document.default_layout : File.read(@layout_filename)
+      block_binding = IsolatedLayoutContext.new.binding_with_block do |section|
+        case section.to_s
+        when 'head'
+          @headers.join("\n")
+        else
+          content
+        end
+      end
+      ERB.new(@layout, 4, '<>').result(block_binding.taint) # $SAFE = 4
+    end
+    
+    #
+    # Formats a hash as html attributes for insertion into a tag.
+    #
+    def html_attrs(attrs)
+      list = attrs.to_a.map {|k,v| "#{k}=\"#{v}\"" }
+      list.join(' ')
+    end
+    
+    #
+    # Add another header to the array of headers.
+    #
+    def add_header(h)
+      # Can't use Array#push in $SAFE = 4 
+      @headers = @headers + [h]
+      h
+    end
+    
+  end
+  
+end

data/test/test_data_doc.rb

@@ -1,11 +1,10 @@
-require File.dirname(__FILE__) + '/test_helper.rb'
+# require File.dirname(__FILE__) + '/test_helper.rb'
+require_relative 'test_helper.rb'
 
-class TestDataDoc < Test::Unit::TestCase
-
-  def setup
-  end
+describe DataDoc do
   
-  def test_truth
-    assert true
+  describe "integration tests" do
+        
   end
+  
 end

data/test/test_data_doc_cli.rb

@@ -1,14 +1,99 @@
-require File.join(File.dirname(__FILE__), "test_helper.rb")
-require 'data_doc/cli'
-
-class TestDataDocCli < Test::Unit::TestCase
-  def setup
-    DataDoc::CLI.execute(@stdout_io = StringIO.new, [])
-    @stdout_io.rewind
-    @stdout = @stdout_io.read
+require_relative 'test_helper.rb'
+require_relative '../lib/data_doc/cli'
+require 'tempfile'
+require 'set'
+
+describe DataDoc::CLI do
+         
+  def execute_cli(*args) 
+    DataDoc::CLI.execute(stdout_io = StringIO.new, args)
+    stdout_io.rewind
+    stdout_io.read.strip
   end
+
+  describe "with no file" do
+
+    it "should present version" do
+      execute_cli('--version').must_equal DataDoc::VERSION
+    end
+
+    it "should present help" do
+      result = execute_cli('--help')
+      result.must_match(/#{DataDoc::DESCRIPTION}/)
+      result.must_match(/Usage:/)
+      result.must_match(/--help/)
+    end
+
+  end
+  
+  describe "with a file" do
+        
+    describe "which is empty" do
+      
+      before do
+        @filename = temp_file("")
+      end
+      
+      it "should return" do
+        execute_cli(@filename)
+      end
   
-  def test_print_default_output
-    assert_match(/To update this executable/, @stdout)
+      it "should require a filename for connection option" do
+        proc { execute_cli('--connection') }.must_raise OptionParser::MissingArgument
+      end
+  
+      describe "with connection settings file" do
+          
+        before do
+          @db_filename = temp_file("")
+          connection_yaml = <<YAML
+adapter: sqlite3
+database: #{@db_filename}
+YAML
+          @connection_filename = temp_file(connection_yaml)
+        end
+        
+        it "should accept a connection option" do
+          execute_cli("--connection", "#{@connection_filename}")
+        end
+      
+      end
+
+      it "should accept a read-only option" do
+        execute_cli("--read-only", @filename)
+      end
+
+      it "should accept a data-only option" do
+        execute_cli("--data-only", @filename)
+      end
+
+      it "should accept an output file option" do
+        output_filename = temp_file("")
+        execute_cli("--output", output_filename, @filename)
+      end
+
+      it "should require a filename for output option" do
+        proc { execute_cli("--output") }.must_raise OptionParser::MissingArgument
+      end
+
+      it "should accept an html format option" do
+        execute_cli("--format", 'html', @filename)
+      end
+
+      it "should require a format for format option" do
+        proc { execute_cli("--format") }.must_raise OptionParser::MissingArgument
+      end
+
+      it "should require a valid format for format option" do
+        proc { execute_cli("--format", 'invalid') }.must_raise OptionParser::InvalidArgument
+      end
+
+      it "should accept a verbose option" do
+        execute_cli("--verbose", @filename)
+      end
+      
+    end
+
   end
+  
 end

data/test/test_data_doc_document.rb

@@ -0,0 +1,76 @@
+require_relative 'test_helper.rb'
+
+describe DataDoc::Document do
+    
+  before do
+    @doc = DataDoc::Document.new
+    @input = ""
+    @expected_output = ""
+  end
+
+  after do
+    output = StringIO.new
+    @doc.output = output
+    @doc.generate(StringIO.new(@input))
+    output.rewind
+    output.read.strip.must_equal @expected_output.strip
+  end
+      
+  it "should process empty input" do
+    @input = ""
+    @doc.layout = temp_file('<%= yield %>')
+    @expected_output = ""
+  end
+  
+  it "should process markdown into html" do
+    @input = '# Introduction'
+    @doc.layout = temp_file('<%= yield %>')
+    @expected_output = '<h1>Introduction</h1>'
+  end
+  
+  describe "layout" do
+
+    it "should wrap the generated text in a layout" do
+      layout_filename = temp_file('layout:<%= yield %>:layout')
+      @input = "<% set_layout_file '#{layout_filename}' %>Some text."
+      @expected_output = "layout:<p>Some text.</p>\n:layout"
+    end
+  
+    it "should wrap the content in a default html layout" do
+      @input = "<p>XXX</p>"
+      @expected_output = erb(DataDoc::Document.default_layout) do |section|
+        @input + "\n" if section.to_s != 'head'
+      end    
+    end
+    
+  end
+  
+  describe "head tags" do
+  
+    it "should insert meta tags into the head" do
+      @input = '<% meta author: "Author" %>'
+      @doc.layout = temp_file('<%= yield :head %>')
+      @expected_output = '<meta author="Author">'
+    end
+  
+    it "should insert script tags into the head" do
+      @input = '<% script(src: "jquery.js") { "XXX" } %>'
+      @doc.layout = temp_file('<%= yield :head %>')
+      @expected_output = "<script src=\"jquery.js\">\nXXX\n</script>"
+    end
+  
+    it "should insert link tags into the head" do
+      @input = '<% link rel: "stylesheet", type: "text/css", href: "mystyle.css" %>'
+      @doc.layout = temp_file('<%= yield :head %>')
+      @expected_output = '<link rel="stylesheet" type="text/css" href="mystyle.css">'
+    end
+  
+    it "should set the html document title in the head and in a div.title in the content" do
+      @input = '<%= title "Title" %>'
+      @doc.layout = temp_file('Head:<%= yield :head %>Content:<%= yield %>')
+      @expected_output = 'Head:<title>Title</title>Content:<div class="title">Title</div>'
+    end
+  
+  end
+
+end

data/test/test_helper.rb

@@ -1,3 +1,21 @@
 require 'stringio'
-require 'test/unit'
+require 'set'
+require 'tempfile'
+require 'minitest/autorun'
+require 'minitest/mock'
+
 require File.dirname(__FILE__) + '/../lib/data_doc'
+
+def temp_file(content)
+  @temp_files ||= Set.new
+  f = Tempfile.new('test_data_doc')
+  @temp_files.add(f)
+  f.write(content)
+  f.rewind
+  f.path
+end
+
+# allows yield to call block
+def erb(content)
+  ERB.new(content).result(binding)
+end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: data_doc
 version: !ruby/object:Gem::Version
-  version: 0.0.2
+  version: 0.0.3
   prerelease: 
 platform: ruby
 authors:
@@ -9,8 +9,24 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2012-10-18 00:00:00.000000000 Z
+date: 2012-10-21 00:00:00.000000000 Z
 dependencies:
+- !ruby/object:Gem::Dependency
+  name: activerecord
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: 2.0.2
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: 2.0.2
 - !ruby/object:Gem::Dependency
   name: rdoc
   requirement: !ruby/object:Gem::Requirement
@@ -62,7 +78,7 @@ dependencies:
 description: ! "Complex documents can be challenging to author, and time-consuming
   to achieve \nhigh levels of internal consistency. This gem offers a Domain Specific
   Language \nto embed structured data in a markdown document and then render it into
-  \nconfigurable tables."
+  \nconfigurable tables. \n\nNote: See security implications below."
 email:
 - gems@shortepic.com
 executables:
@@ -71,27 +87,31 @@ extensions: []
 extra_rdoc_files:
 - History.txt
 - Manifest.txt
-- PostInstall.txt
 - README.rdoc
 files:
 - History.txt
 - Manifest.txt
-- PostInstall.txt
 - README.rdoc
 - Rakefile
 - bin/data_doc
 - lib/data_doc.rb
 - lib/data_doc/cli.rb
+- lib/data_doc/document.rb
+- lib/data_doc/store.rb
+- lib/data_doc/present.rb
 - script/console
 - script/destroy
 - script/generate
 - test/test_data_doc.rb
+- test/test_data_doc_document.rb
+- test/test_data_doc_store.rb
+- test/test_data_doc_present.rb
 - test/test_data_doc_cli.rb
 - test/test_helper.rb
 - .gemtest
 homepage: http://github.com/alilee/data_doc
 licenses: []
-post_install_message: PostInstall.txt
+post_install_message: 
 rdoc_options:
 - --main
 - README.rdoc
@@ -105,7 +125,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
       version: '0'
       segments:
       - 0
-      hash: -1827853280249206312
+      hash: -2005409692758115623
 required_rubygems_version: !ruby/object:Gem::Requirement
   none: false
   requirements:
@@ -122,4 +142,7 @@ summary: Complex documents can be challenging to author, and time-consuming to a
 test_files:
 - test/test_data_doc.rb
 - test/test_data_doc_cli.rb
+- test/test_data_doc_document.rb
+- test/test_data_doc_present.rb
+- test/test_data_doc_store.rb
 - test/test_helper.rb

data/PostInstall.txt

@@ -1,7 +0,0 @@
-
-For more information on data_doc, see http://data_doc.rubyforge.org
-
-NOTE: Change this information in PostInstall.txt 
-You can also delete it if you don't want it.
-
-