slaw 0.7.3 → 0.8.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: ea560c6ca4986938adc55f944e1530e2f213af40
-  data.tar.gz: 27c25b8edf02ca79a169f3ba8ae6e8215f44c716
+  metadata.gz: 26b655c478bd057b9c89b7a7972984e387f9c447
+  data.tar.gz: 9c352182961b3c6a8dc6676c2de43e71777dc7ba
 SHA512:
-  metadata.gz: 6c456e3373e43272b6ec6c8e15bc848c61e479cca955ff454daf06949c947ddc6b26b02b2f3d995663e608a434bea91553d60b13aa5a31965bd2509ad2840c0e
-  data.tar.gz: 38741a5d5cd8a9a5c72b9959719f6051c0e0ae88b56a8be50bd6fb3bc7f2e67c4121a3ec604047209f22be3ee46764c3898f0df8bbbeaa9630b4da63f634dce6
+  metadata.gz: f1d882e1bbe07aa2ef08fcbeb5d82eb25aa4caede00b876a342ca050f89f4c184699c65e9a47787ceed1750644d9c634cb1281a0e070fad6f933bdca38ee6c23
+  data.tar.gz: 823fee99147547c3683685f443b6bd72b755fe840013059fa0ffecd02fae3a0bc3d0ccfcb66ecc6f419ba0aa2dbe42b44c27738f18db111232723a95fc169107

data/README.md

@@ -216,6 +216,15 @@ Akoma Ntoso `component` elements at the end of the XML document, with a name of
 
 ## Changelog
 
+### 0.8.0
+
+* FEATURE: parse command only reformats input for PDFs or when --reformat is given
+* FIX: don't error on defn tags without link to defined term
+
+### 0.7.4
+
+* use refersTo to identify blocks containing term definitions, rather than setting an (invalid) ID
+
 ### 0.7.3
 
 * add link-definitions command to find and extract defined terms and link them to their definitions

data/bin/slaw

@@ -21,6 +21,7 @@ class SlawCLI < Thor
   option :fragment, type: :string, desc: "Akoma Ntoso element name that the imported text represents. Support depends on the grammar."
   option :id_prefix, type: :string, desc: "Prefix to be used when generating ID elements when parsing a fragment."
   option :section_number_position, enum: ['before-title', 'after-title', 'guess'], desc: "Where do section titles come in relation to the section number? Default: before-title"
+  option :reformat, type: :boolean, desc: "Reformat common formatting issues to make grammar matching better. Default: true for PDF files, false otherwise"
   def parse(name)
     logging
 
@@ -34,6 +35,7 @@ class SlawCLI < Thor
     case options[:input]
     when 'pdf'
       text = extractor.extract_from_pdf(name)
+      options[:reformat] = true if options[:reformat].nil?
     when 'text'
       text = extractor.extract_from_text(name)
     else
@@ -42,6 +44,8 @@ class SlawCLI < Thor
 
     generator = Slaw::ActGenerator.new
 
+    text = generator.reformat(text) if options[:reformat]
+
     if options[:fragment]
       generator.document_class = Slaw::Fragment
 

data/lib/slaw/extract/extractor.rb

@@ -17,14 +17,7 @@ module Slaw
 
       @@pdftotext_path = "pdftotext"
 
-      # Object with text cleaning helpers
-      attr_accessor :cleanser
-
-      def initialize
-        @cleanser = Slaw::Parse::Cleanser.new
-      end
-
-      # Extract text from a file and run cleanup on it.
+      # Extract text from a file.
       #
       # @param filename [String] filename to extract from
       #
@@ -61,7 +54,7 @@ module Slaw
 
           case status.exitstatus
           when 0
-            return cleanup(stdout)
+            return stdout
           when 3
             return nil if retried
             retried = true
@@ -82,7 +75,7 @@ module Slaw
       end
 
       def extract_from_text(filename)
-        cleanup(File.read(filename))
+        File.read(filename)
       end
 
       # Extract text from +filename+ by sending it to apache tika
@@ -99,15 +92,6 @@ module Slaw
         text
       end
 
-      # Run general once-off cleanup of extracted text.
-      def cleanup(text)
-        text = @cleanser.cleanup(text)
-        text = @cleanser.remove_empty_lines(text)
-        text = @cleanser.reformat(text)
-
-        text
-      end
-
       def remove_pdf_password(filename)
         file = Tempfile.new('steno')
         begin

data/lib/slaw/generator.rb

@@ -30,10 +30,18 @@ module Slaw
       act
     end
 
+    # Run basic cleanup on text, such as ensuring clean newlines
+    # and removing tabs. This is always automatically done before
+    # processing.
     def cleanup(text)
-      text = @cleanser.cleanup(text)
-      text = @cleanser.reformat(text)
-      text
+      @cleanser.cleanup(text)
+    end
+
+    # Reformat some common errors in text to help make parsing more
+    # successful. Option and only recommended when processing a document
+    # for the first time.
+    def reformat(text)
+      @cleanser.reformat(text)
     end
 
     # Try to determine if section numbers come after titles,

data/lib/slaw/parse/builder.rb

@@ -222,16 +222,30 @@ module Slaw
         terms = {}
         doc.xpath('//a:def', a: NS).each do |defn|
           # <p>"<def refersTo="#term-affected_land">affected land</def>" means land in respect of which an application has been lodged in terms of section 17(1);</p>
-          id = defn['refersTo'].sub(/^#/, '')
-          term = defn.content
-          terms[id] = term
+          if defn['refersTo']
+            id = defn['refersTo'].sub(/^#/, '')
+            term = defn.content
+            terms[id] = term
 
-          logger.info("+ Found definition for: #{term}")
+            logger.info("+ Found definition for: #{term}")
+          end
         end
 
         terms
       end
 
+      # Find defined terms in the document.
+      #
+      # This looks for heading elements with the words 'definitions' or 'interpretation',
+      # and then looks for phrases like
+      #
+      #   "this word" means something...
+      #
+      # It identifies "this word" as a defined term and wraps it in a def tag with a refersTo
+      # attribute referencing the term being defined. The surrounding block
+      # structure is also has its refersTo attribute set to the term. This way, the term
+      # is both marked as defined, and the container element with the full
+      # definition of the term is identified.
       def guess_at_definitions(doc)
         doc.xpath('//a:section', a: NS).select do |section|
           # sections with headings like Definitions
@@ -254,16 +268,17 @@ module Slaw
               term_id = 'term-' + term.gsub(/[^a-zA-Z0-9_-]/, '_')
 
               # <p>"<def refersTo="#term-affected_land">affected land</def>" means land in respect of which an application has been lodged in terms of section 17(1);</p>
-              defn = doc.create_element('def', term, refersTo: "##{term_id}")
+              refersTo = "##{term_id}"
+              defn = doc.create_element('def', term, refersTo: refersTo)
               rest = match.post_match
 
               text.before(defn)
               defn.before(doc.create_text_node('"'))
               text.content = '"' + rest
 
-              # adjust the container's id
-              parent = find_up(container, ['blockList', 'point']) || find_up(container, ['subsection', 'section'])
-              parent['id'] = "def-#{term_id}"
+              # adjust the container's refersTo attribute
+              parent = find_up(container, ['item', 'point', 'blockList', 'list', 'paragraph', 'subsection', 'section', 'chapter', 'part'])
+              parent['refersTo'] = refersTo
             end
           end
         end

data/lib/slaw/version.rb

@@ -1,3 +1,3 @@
 module Slaw
-  VERSION = "0.7.3"
+  VERSION = "0.8.0"
 end

data/spec/extract/extractor_spec.rb

@@ -9,6 +9,6 @@ describe Slaw::Extract::Extractor do
     f.write('This is some text')
     f.rewind
 
-    subject.extract_from_file(f.path).should == "This is some text\n"
+    subject.extract_from_file(f.path).should == "This is some text"
   end
 end

data/spec/parse/builder_spec.rb

@@ -617,14 +617,14 @@ XML
       subject.guess_at_definitions(doc)
       doc.to_s.should == section(<<XML
         <heading>Definitions</heading>
-        <subsection id="def-term-authorised_official">
+        <subsection id="section-1.subsection-1" refersTo="#term-authorised_official">
           <content>
             <p>"<def refersTo="#term-authorised_official">authorised official</def>" means any official of the Council who has been authorised by it to administer, implement and enforce the provisions of these By-laws;</p>
           </content>
         </subsection>
         <subsection id="section-1.subsection-2">
           <content>
-            <blockList id="def-term-Council">
+            <blockList id="section-1.subsection-2.list2" refersTo="#term-Council">
               <listIntroduction>"<def refersTo="#term-Council">Council</def>" means –                </listIntroduction>
               <item id="section-1.subsection-2.list2.a">
                 <num>(a)</num>

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: slaw
 version: !ruby/object:Gem::Version
-  version: 0.7.3
+  version: 0.8.0
 platform: ruby
 authors:
 - Greg Kempe
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-08-25 00:00:00.000000000 Z
+date: 2015-09-06 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler