slaw 1.0.0.alpha.3 → 1.0.0.alpha.4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 31ca69bdfe68c0e688320a033b63bc319dd3494e
-  data.tar.gz: 95abf4fbf34526c2497f87413bfb1a9bc0229e4c
+  metadata.gz: a2fbb349a6f18f3b78feda6a8242e3688db6f65b
+  data.tar.gz: 6dc113e8c0194a813b5a0837fc6bb5dabdfe7570
 SHA512:
-  metadata.gz: ccab43b362cd4eb771a5eeed3700b24ef7cc957f6f13144e461e79907518206f630c256fe442259224ddf76c02985743bedf1be6686a118a3ff97f909d1a4e80
-  data.tar.gz: 192cad74ecbc0c26335b2eda024ff7f206171f32fdfb4c6823212744f4aca463f21d97878e5f718ab2695ce9d37784bf1bdd4ec2cd494fd3a3daa0d32d2443e4
+  metadata.gz: a3941935634b014c79358c16784acfb32ac16f6fd083b4a9f31a029fb90d0f19eff4ca4eca29e71e54348a9458617c53a661b0537d2b4922157dba6fdd64f7ff
+  data.tar.gz: f76579ebb6a935e1c46a3ef4ba5eaafe3761673408fc35006c5036d677650aeaa534335095bff3ca139bb8e09a5df8bf384dc4071f432d5c5053a933391e76bf

data/README.md

@@ -223,6 +223,7 @@ Akoma Ntoso `component` elements at the end of the XML document, with a name of
 * Improved support for other legal traditions / grammars.
 * Add Polish legal tradition grammar.
 * Slaw no longer does too much introspection of a parsed document, since that can be so tradition-dependent.
+* Move reformatting out of Slaw since it's tradition-dependent.
 * Remove definition linking, Slaw no longer supports it.
 
 ### 0.17.2

data/bin/slaw

@@ -20,7 +20,6 @@ class SlawCLI < Thor
   option :fragment, type: :string, desc: "Akoma Ntoso element name that the imported text represents. Support depends on the grammar."
   option :id_prefix, type: :string, desc: "Prefix to be used when generating ID elements when parsing a fragment."
   option :section_number_position, enum: ['before-title', 'after-title', 'guess'], desc: "Where do section titles come in relation to the section number? Default: before-title"
-  option :reformat, type: :boolean, desc: "Reformat common formatting issues to make grammar matching better. Default: true for PDF files, false otherwise"
   option :crop, type: :string, desc: "Crop box for PDF files, as 'left,top,width,height'."
   option :grammar, type: :string, desc: "Grammar name (usually a two-letter country code). Default is za."
   def parse(name)
@@ -39,7 +38,6 @@ class SlawCLI < Thor
     case options[:input]
     when 'pdf'
       text = extractor.extract_from_pdf(name)
-      options[:reformat] = true if options[:reformat].nil?
     when 'text'
       text = extractor.extract_from_text(name)
     else
@@ -48,8 +46,6 @@ class SlawCLI < Thor
 
     generator = Slaw::ActGenerator.new(options[:grammar] || 'za')
 
-    text = generator.reformat(text) if options[:reformat]
-
     if options[:fragment]
       generator.document_class = Slaw::Fragment
 

data/lib/slaw/parse/cleanser.rb

@@ -14,23 +14,11 @@ module Slaw
       def cleanup(s)
         s = scrub(s)
         s = correct_newlines(s)
-        s = fix_quotes(s)
         s = expand_tabs(s)
         s = chomp(s)
         s = enforce_newline(s)
       end
 
-      # Run deeper introspections and reformat the text, such as
-      # unwrapping/re-wrapping lines. These may not be safe to run
-      # multiple times.
-      def reformat(s)
-        s = remove_boilerplate(s)
-        s = unbreak_lines(s)
-        s = break_lines(s)
-        s = strip_toc(s)
-        s = enforce_newline(s)
-      end
-
       # ------------------------------------------------------------------------
 
       def remove_empty_lines(s)
@@ -51,36 +39,12 @@ module Slaw
          .gsub(/–/, '-')
       end
 
-      # change weird quotes to normal ones
-      def fix_quotes(s)
-        s.gsub(/‘‘|’’|''/, '"')
-      end
-
       # tabs to spaces
       def expand_tabs(s)
         s.gsub(/\t/, ' ')\
          .gsub("\u00A0", ' ') # non-breaking space
       end
 
-      # Try to remove boilerplate lines found in many files, such as page numbers.
-      def remove_boilerplate(s)
-        # nuke any line to do with Sabinet and the government printer
-        s.gsub(/^.*Sabinet.*Government Printer.*$/i, '')\
-         .gsub(/^.*Provincial Gazette \d+.*$/i, '')\
-         .gsub(/^.*Provinsiale Koerant \d+.*$/i, '')\
-         .gsub(/^.*PROVINCIAL GAZETTE.*$/, '')\
-         .gsub(/^.*PROVINSIALE KOERANT.*$/, '')\
-         .gsub(/^\s*\d+\s*$/, '')\
-         .gsub(/^.*This gazette is also available.*$/, '')\
-        # get rid of date lines
-         .gsub(/^\d{1,2}\s+\w+\s+\d{4}$/, '')\
-        # get rid of page number lines
-         .gsub(/^\s*page \d+( of \d+)?\s*\n/i, '')\
-         .gsub(/^\s*\d*\s*No\. \d+$/, '')\
-        # get rid of lines with lots of ____ or ---- chars, they're usually pagebreaks
-         .gsub(/^.*[_-]{5}.*$/, '')
-      end
-
       # Get rid of whitespace at the end of lines and at the start and end of the 
       # entire string.
       def chomp(s)
@@ -95,106 +59,6 @@ module Slaw
         # ensure string ends with a newline
         s.end_with?("\n") ? s : (s + "\n")
       end
-
-      # Make educated guesses about lines that should
-      # have been broken but haven't, and break them.
-      #
-      # This is very dependent on a locale's legislation grammar, there are
-      # lots of rules of thumb that make this work.
-      def break_lines(s)
-        # often we find a section title munged onto the same line as its first statement
-        # eg:
-        # foo bar. New section title 62. (1) For the purpose
-        s = s.gsub(/\. ([^.]+) (\d+\. ?\(1\) )/, ".\n" + '\1' + "\n" + '\2')
-
-        # New section title 62. (1) For the purpose
-        s = s.gsub(/(\w) (\d+\. ?\(1\) )/, '\1' + "\n" + '\2')
-
-        # (1) foo; (2) bar
-        # (1) foo. (2) bar
-        s = s.gsub(/(\w{3,}[;.]) (\([0-9a-z]+\))/, "\\1\n\\2")
-
-        # (1) foo; and (2) bar
-        # (1) foo; or (2) bar
-        s = s.gsub(/; (and|or) \(/, "; \\1\n(")
-
-        # The officer-in-Charge may – (a) remove all withered natural... \n(b)
-        # We do this last, because by now we should have reconised that (b) should already
-        # be on a new line.
-        s = s.gsub(/ (\(a\) .+?\n\(b\))/, "\n\\1")
-
-        # "foo" means ...; "bar" means
-        s = s.gsub(/; (["”“][^"”“]+?["”“] means)/, ";\n\\1")
-
-        # CHAPTER 4 PARKING METER PARKING GROUNDS Place of parking
-        s = s.gsub(/([A-Z0-9 ]{5,}) ([A-Z][a-z ]{5,})/, "\\1\n\\2")
-
-        s
-      end
-
-      # Find likely candidates for unnecessarily broken lines
-      # and unbreaks them.
-      def unbreak_lines(s)
-        lines = s.split(/\n/)
-        output = []
-
-        # set of regex matcher pairs, one for the prev line, one for the current line
-        matchers = [
-          [/[a-z0-9]$/, /^\s*[a-z]/],  # line ends with and starst with lowercase
-          [/;$/, /^\s*(and|or)/],      # ends with ; then and/or on new line
-        ]
-
-        prev = nil
-        lines.each_with_index do |line, i|
-          if i == 0
-            output << line
-          else
-            prev = output[-1]
-            unbreak = false
-
-            for prev_re, curr_re in matchers
-              if prev =~ prev_re and line =~ curr_re
-                unbreak = true
-                break
-              end
-            end
-
-            if unbreak
-              output[-1] = prev + ' ' + line
-            else
-              output << line
-            end
-          end
-        end
-
-        output.join("\n")
-      end
-
-      # Do our best to remove table of contents at the start,
-      # it really confuses the grammer.
-      def strip_toc(s)
-        # first, try to find 'TABLE OF CONTENTS' anywhere within the first 4K of text,
-        if toc_start = s[0..4096].match(/TABLE OF CONTENTS/i)
-
-          # grab the first non-blank line after that, it's our end-of-TOC marker
-          if eol = s.match(/^(.+?)$/, toc_start.end(0))
-            marker = eol[0]
-
-            # search for the first line that is a prefix of marker (or vv), and delete
-            # everything in between
-            posn = eol.end(0)
-            while m = s.match(/^(.+?)$/, posn)
-              if marker.start_with?(m[0]) or m[0].start_with?(marker)
-                return s[0...toc_start.begin(0)] + s[m.begin(0)..-1]
-              end
-
-              posn = m.end(0)
-            end
-          end
-        end
-
-        s
-      end
     end
   end
 end

data/lib/slaw/version.rb

@@ -1,3 +1,3 @@
 module Slaw
-  VERSION = "1.0.0.alpha.3"
+  VERSION = "1.0.0.alpha.4"
 end

data/spec/parse/cleanser_spec.rb

@@ -16,133 +16,4 @@ describe Slaw::Parse::Cleanser do
       subject.expand_tabs("foo \u00A0bar").should == "foo  bar"
     end
   end
-
-  describe '#unbreak_lines' do
-    it 'should unbreak simple lines' do
-      subject.unbreak_lines("""
-8.2.3 an additional fee or tariff, which is
-to be determined by the City in its
-sole discretion, in respect of additional
-costs incurred or services.
-8.3 In the event that a person qualifies for
-a permit, but has motivated in writing
-the inability to pay the fee contemplated.""").should == """
-8.2.3 an additional fee or tariff, which is to be determined by the City in its sole discretion, in respect of additional costs incurred or services.
-8.3 In the event that a person qualifies for a permit, but has motivated in writing the inability to pay the fee contemplated."""
-    end
-
-    it 'should not unbreak section headers' do
-      subject.unbreak_lines("""
-8.4.3 must be a South African citizen, failing which, must be in possession of
-a valid work permit which includes, but is not limited to, a refugee
-permit; and
-8.4.4 must not employ and actively utilise the services of more than 20
-(twenty) persons.""").should == """
-8.4.3 must be a South African citizen, failing which, must be in possession of a valid work permit which includes, but is not limited to, a refugee permit; and
-8.4.4 must not employ and actively utilise the services of more than 20
-(twenty) persons."""
-    end
-  end
-
-  describe '#break_lines' do
-    it 'should break nested lists' do
-      subject.break_lines('stored, if known; (b) the number of trolleys').should == "stored, if known;\n(b) the number of trolleys"
-
-      subject.break_lines("(5) The officer-in-Charge may – (a) remove all withered natural flowers, faded or damaged artificial flowers and any receptacle placed on a grave; or\n(b) 30 days after publishing a general").should == "(5) The officer-in-Charge may –\n(a) remove all withered natural flowers, faded or damaged artificial flowers and any receptacle placed on a grave; or\n(b) 30 days after publishing a general"
-
-      subject.break_lines("(2) No person may – (a) plant, cut or remove plants, shrubs or flowers on a grave without the permission of the officer-in-charge; (b) plant, cut or remove plants, shrubs or flowers on the berm section; or").should == "(2) No person may –\n(a) plant, cut or remove plants, shrubs or flowers on a grave without the permission of the officer-in-charge;\n(b) plant, cut or remove plants, shrubs or flowers on the berm section; or"
-
-      subject.break_lines('(b) its successor in title; or (c) a structure or person exercising a delegated power or carrying out an instruction, where any power in these By-laws, has been delegated or sub-delegated or an instruction given as contemplated in, section 59 of the Local Government: Municipal Systems Act, 2000 (Act No. 32 of 2000); or (d) a service provider fulfilling a responsibility under these By-laws, assigned to it in terms of section 81(2) of the Local Government: Municipal Systems Act, 2000, or any other law, as the case may be;').should == "(b) its successor in title; or\n(c) a structure or person exercising a delegated power or carrying out an instruction, where any power in these By-laws, has been delegated or sub-delegated or an instruction given as contemplated in, section 59 of the Local Government: Municipal Systems Act, 2000 (Act No. 32 of 2000); or\n(d) a service provider fulfilling a responsibility under these By-laws, assigned to it in terms of section 81(2) of the Local Government: Municipal Systems Act, 2000, or any other law, as the case may be;"
-    end
-
-    it 'should break at likely subsections' do
-      subject.break_lines('(c) place a metal cot on any grave. (3) A person may only erect, place or leave, an object or decoration on a grave during the first 30 days following the burial. (4) Natural or artificial flowers contained in receptacles may be placed on a grave at any time, but in a grave within a berm section or with a headstone, such flowers may only be placed in the socket provided. (5) The officer-in-Charge may – (a) remove all withered natural flowers, faded or damaged artificial flowers and any receptacle placed on a grave; or').should == "(c) place a metal cot on any grave.\n(3) A person may only erect, place or leave, an object or decoration on a grave during the first 30 days following the burial.\n(4) Natural or artificial flowers contained in receptacles may be placed on a grave at any time, but in a grave within a berm section or with a headstone, such flowers may only be placed in the socket provided.\n(5) The officer-in-Charge may – (a) remove all withered natural flowers, faded or damaged artificial flowers and any receptacle placed on a grave; or"
-    end
-
-    it 'should break lines at likely section titles' do
-      subject.break_lines('foo bar. New section title 62. (1) For the purpose').should == "foo bar.\nNew section title\n62. (1) For the purpose"
-      subject.break_lines('foo bar. New section title 62.(1) For the purpose').should == "foo bar.\nNew section title\n62.(1) For the purpose"
-      subject.break_lines('New section title 62. (1) For the purpose').should == "New section title\n62. (1) For the purpose"
-      subject.break_lines('New section title 62.(1) For the purpose').should == "New section title\n62.(1) For the purpose"
-    end
-
-    it 'should clean up wrapped definition lines after pdf' do
-      subject.break_lines('“agricultural holding” means a portion of land not less than 0.8 hectares in extent used solely or mainly for the purpose of agriculture, horticulture or for breeding or keeping domesticated animals, poultry or bees; “approved” means as approved by the Council; “bund wall” means a containment wall surrounding an above ground storage tank, constructed of an impervious material and designed to contain 110% of the contents of the tank; “certificate of fitness” means a certificate contemplated in section 20; “certificate of registration” means a certificate contemplated in section 35;').should == "“agricultural holding” means a portion of land not less than 0.8 hectares in extent used solely or mainly for the purpose of agriculture, horticulture or for breeding or keeping domesticated animals, poultry or bees;\n“approved” means as approved by the Council;\n“bund wall” means a containment wall surrounding an above ground storage tank, constructed of an impervious material and designed to contain 110% of the contents of the tank;\n“certificate of fitness” means a certificate contemplated in section 20;\n“certificate of registration” means a certificate contemplated in section 35;"
-    end
-
-    it 'should break at CAPCASE TO Normal Case' do
-      subject.break_lines('CHAPTER 3 PARKING METER PARKING GROUNDS Place of parking 7. No person may park or cause or permit to be parked any vehicle or allow a vehicle to be or remain in a parking meter parking ground otherwise than in a parking bay.').should == "CHAPTER 3 PARKING METER PARKING GROUNDS\nPlace of parking 7. No person may park or cause or permit to be parked any vehicle or allow a vehicle to be or remain in a parking meter parking ground otherwise than in a parking bay."
-    end
-  end
-
-  describe '#strip_toc' do
-    it 'should handle no toc' do
-      s = "City of Johannesburg Metropolitan Municipality
-CULTURE AND RECREATION BY-LAWS ( )PUBLISHED IN PROVINCIAL GAZETTE EXTRAORDINARY NO 179 DATED 21 MAY 2004 UNDER NOTICE NUMBER 825
-
-CITY OF JOHANNESBURG METROPOLITAN MUNICIPALITY
-CULTURE AND RECREATION BY-LAWS
-The Municipal Manager of the City of Johannesburg Metropolitan Municipality hereby, in terms of Section 13(a) of the Local Government: Municipal Systems Act, 2000 (Act No. 32 of 2000), publishes the Culture and RecreationBy-laws for the City of Johannesburg Metropolitan Municipality, as approved by its Council, as set out hereunder.
-CITY OF JOHANNESBURG METROPOLITAN MUNICIPALITY
-CULTURE AND RECREATION BY-LAWS
-CHAPTER 1 LIBRARY AND INFORMATION SERVICES
-Definitions and interpretation
-1. (1) In this Chapter, unless the context otherwise indicates-"
-      subject.strip_toc(s).should == s
-    end
-
-    it 'should strip table of contents' do
-      subject.strip_toc("City of Johannesburg Metropolitan Municipality
-CULTURE AND RECREATION BY-LAWS ( )PUBLISHED IN PROVINCIAL GAZETTE EXTRAORDINARY NO 179 DATED 21 MAY 2004 UNDER NOTICE NUMBER 825
-
-CITY OF JOHANNESBURG METROPOLITAN MUNICIPALITY
-CULTURE AND RECREATION BY-LAWS
-The Municipal Manager of the City of Johannesburg Metropolitan Municipality hereby, in terms of Section 13(a) of the Local Government: Municipal Systems Act, 2000 (Act No. 32 of 2000), publishes the Culture and RecreationBy-laws for the City of Johannesburg Metropolitan Municipality, as approved by its Council, as set out hereunder.
-CITY OF JOHANNESBURG METROPOLITAN MUNICIPALITY
-CULTURE AND RECREATION BY-LAWS
-TABLE OF CONTENTS
-CHAPTER 1
-LIBRARY AND INFORMATION SERVICES
-1. Definitions and interpretation 2. Admission to library buildings 3. Membership 4. Loan of library material 5. Return of library material 6. Overdue library material 7. Reservation of library material 8. Lost and damaged library material 9. Handling of library material 10. Exposure of library material to notifiable and infectious diseases 11. Library material for special and reference purposes 12. Reproduction of library material and objects and use of facsimile facilities 13. Library hours 14. Hire and use of auditoria and lecture rooms or library space 15. Internet viewing stations 16. Hiring of multimedia library space 17. Performing arts library 18. Availability of By-laws and notices in a library 19. Conduct in the libraries
-CHAPTER 2
-ARTS AND CULTURE AND COMMUNITY CENTRE FACILITIES
-
-Part 1: Hire and use of community arts & culture facilities
-20. Definitions and interpretation 21. Rights and status of artists 22. Co-operation between Council departments 23. Application for hiring of premises 24. Prescribed fees 25. Payment of fees 26. Period of hire 27. Adjustment of period of hire 28. Joint hire 29. Sub-letting 30. Condition of premises 31. Duties of the hirer 32. Advertisements and decorations 33. Admissions and sale of tickets 34. Overcrowding 35. Sale of refreshments 36. Services 37. Cancellation due to destruction of premises 38. Cancellation due to non-compliance 39. Termination of period of hire 40 Fire hazards and Insurance 41. Storage facilities 42. Equipment 43. Right of entry 44. Inspection 45. Regulations 46. Nuisance
-Part 2: Community centres
-47. Group activities 48. Membership 49. Membership fees 50. Use of centres for religious or personal purposes 51. Dress code 52. Conduct of children 53. Application of certain sections of part 1 of Chapter 2 to centres 54. Application of certain sections of Part 2 of Chapter 3 to centres
-
-CHAPTER 3
-RECREATION AND SPORT
-Part 1: Camping and caravan parks
-55. Definitions and interpretation 56. Lighting of fires prohibited 57. Permits 58. Extension of permits 59. Limitation on the period of occupancy of a camping site 60. Allocation and Use of sites 61. Proper use of roads and pathways 62. Reservation of sites 63. Right of refusal to issue or renew permits 64. Obligations of permit holders 65. Cancellation of permits 66. Access and loitering by members of the public prohibited 67. Site to be left in a clean condition 68. Washing of clothes and utensils and preparation of foodstuffs 69. Trading without permission 70. Damage to Vegetation or Property 71. Instructions of camping officer to be complied with 72 Registration and use of firearms 73. Protection of wildlife 74. Special requirements regarding caravan parks and caravans
-Part 2: Sport Facilities
-75. Definitions and interpretation 76. Administration 77. Access conditions 78. Smoking 79. Alcoholic beverages 80. Duties of hirer 81. Dress code 82. Hiring of sport facilities 83. Reservation of sport facilities by the Council 84. Group activities 85. Public decency 86. Clothing and personal effects 87. Prescribed fees 88. Generally prohibited conduct 89. Animals 90. Infectious Diseases 91. Firearms and Traditional Weapons 92. Disturbance by sound systems 93. Sale of food and refreshments 94. Filming and photographs 95. Sport advisory forum
-
-CHAPTER 4 MISCELLANEOUS 96. Definitions and interpretation 97. Animals in facilities 98. Liability for acts and omissions 99. Offences and penalties 100.
-Repeal
-SCHEDULE 1 BY-LAWS REPEALED
-CHAPTER 1 LIBRARY AND INFORMATION SERVICES
-Definitions and interpretation
-1. (1) In this Chapter, unless the context otherwise indicates-").should == "City of Johannesburg Metropolitan Municipality
-CULTURE AND RECREATION BY-LAWS ( )PUBLISHED IN PROVINCIAL GAZETTE EXTRAORDINARY NO 179 DATED 21 MAY 2004 UNDER NOTICE NUMBER 825
-
-CITY OF JOHANNESBURG METROPOLITAN MUNICIPALITY
-CULTURE AND RECREATION BY-LAWS
-The Municipal Manager of the City of Johannesburg Metropolitan Municipality hereby, in terms of Section 13(a) of the Local Government: Municipal Systems Act, 2000 (Act No. 32 of 2000), publishes the Culture and RecreationBy-laws for the City of Johannesburg Metropolitan Municipality, as approved by its Council, as set out hereunder.
-CITY OF JOHANNESBURG METROPOLITAN MUNICIPALITY
-CULTURE AND RECREATION BY-LAWS
-CHAPTER 1 LIBRARY AND INFORMATION SERVICES
-Definitions and interpretation
-1. (1) In this Chapter, unless the context otherwise indicates-"
-    end
-  end
-
-  describe '#remove_boilerplate' do
-    it 'should handle no toc' do
-      s = "(2)The provisions of section 12 (1) (a), (b), (d) and (g) and section 12(2), (3), (4) and (5), read with the necessary changes, apply to the taking into custody of cats.\nClaiming of impounded dogs and cats\n_____________________________________________________________________________________________ ___________ By-laws relating to Dogs and Cats for Promulgation\n14. (1) Any person may claim an impounded dog or cat if he or she –\n(a) satisfies the poundmaster that he or she is the owner or is otherwise entitled to the custody of the dog or cat concerned;"
-
-      subject.remove_boilerplate(s).should == "(2)The provisions of section 12 (1) (a), (b), (d) and (g) and section 12(2), (3), (4) and (5), read with the necessary changes, apply to the taking into custody of cats.\nClaiming of impounded dogs and cats\n\n14. (1) Any person may claim an impounded dog or cat if he or she –\n(a) satisfies the poundmaster that he or she is the owner or is otherwise entitled to the custody of the dog or cat concerned;"
-    end
-  end
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: slaw
 version: !ruby/object:Gem::Version
-  version: 1.0.0.alpha.3
+  version: 1.0.0.alpha.4
 platform: ruby
 authors:
 - Greg Kempe