slaw 0.14.2 → 0.15.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 00d35b04f6d4d01a206bf5425face038e0b86f56
-  data.tar.gz: 514c2b482ffc969b32bdcd7e0d48cce44f63f979
+  metadata.gz: 96b892b9bbe4ff26ec5609579851b092bb32aa9b
+  data.tar.gz: fe30f6724d2c2bb282d3df0f173a749451f3e291
 SHA512:
-  metadata.gz: f4e0f3eda7310ad7bc60ff57e19e7151f55d4736fb9beffabe560311c97a611fe870833593f96c7ac7829c1deec2e31f91e2d1c54729f8f1c8ced9fdcb369408
-  data.tar.gz: d48951afdc9be70b291451356a1e560a4d10bf8ce03e06387ce6c87c8a635878bb3b57e8fd1a3f0df802005af7ad0366e1286c8623866ca5fef9f85f854b3ae1
+  metadata.gz: 977c23bc3650313e40d7fa7358dd848d51011da6329f3b4d905f388b6c79bfbbcf488b27ddd68b5f821365e3a1b1cfebc39f78d52c05257579f484ab2985714c
+  data.tar.gz: '0528d773d487b6cde6e5b67c3a4aa69f89c389e2ec62136c0495ec7df9f5278fba5023f9aac33b7bb542a5f795989b163f3d109fd558e484fed2d5848fd7437e'

data/README.md

@@ -218,6 +218,10 @@ Akoma Ntoso `component` elements at the end of the XML document, with a name of
 
 ## Changelog
 
+### 0.15.0
+
+* Support tables in many non-PDF documents (eg. Word documents) by converting to HTML and then to Akoma Ntoso
+
 ### 0.14.2
 
 * Convert non-breaking space (\xA0) to space

data/lib/slaw/extract/extractor.rb

@@ -28,6 +28,8 @@ module Slaw
         case mimetype && mimetype.type
         when 'application/pdf'
           extract_from_pdf(filename)
+        when 'text/html', nil
+          extract_from_html(filename)
         when 'text/plain', nil
           extract_from_text(filename)
         else
@@ -78,6 +80,10 @@ module Slaw
         File.read(filename)
       end
 
+      def extract_from_html(filename)
+        html_to_text(File.read(filename))
+      end
+
       # Extract text from +filename+ by sending it to apache tika
       # http://tika.apache.org/
       def extract_via_tika(filename)
@@ -87,9 +93,19 @@ module Slaw
         require 'slaw/extract/yomu_patch'
         logger.info("Using Tika to get text from #{filename}. You need a JVM installed for this.")
 
-        text = Yomu.text_from_file(filename)
-        logger.info("Tika returned #{text.length} bytes")
-        text
+        html = Yomu.text_from_file(filename)
+        logger.info("Tika returned #{html.length} bytes")
+        # transform html into text
+        html_to_text(html)
+      end
+
+      def html_to_text(html)
+        here = File.dirname(__FILE__)
+        xslt = Nokogiri::XSLT(File.open(File.join([here, 'html_to_akn_text.xsl'])))
+
+        text = xslt.transform(Nokogiri::HTML(html)).to_s
+        # remove XML encoding at top
+        text.sub(/^<\?xml [^>]*>/, '')
       end
 
       def remove_pdf_password(filename)

data/lib/slaw/extract/html_to_akn_text.xsl

@@ -0,0 +1,115 @@
+<?xml version="1.0"?>
+<xsl:stylesheet xmlns:xsl="http://www.w3.org/1999/XSL/Transform" version="1.0"
+  xmlns="http://www.w3.org/1999/xhtml">
+
+  <xsl:output method="text" indent="no" omit-xml-declaration="yes" encoding="utf-8" />
+  <xsl:strip-space elements="*"/>
+
+  <xsl:template match="html">
+    <xsl:apply-templates/>
+  </xsl:template>
+
+  <xsl:template match="head|style|script|link" />
+
+  <xsl:template match="p|div">
+    <xsl:choose>
+      <xsl:when test="starts-with(., '[') and substring(., string-length(.)) = ']'">
+        <!-- block elems that are wrapped in [ and ] are probably remarks -->
+        <xsl:text>[</xsl:text><xsl:apply-templates /><xsl:text>]</xsl:text>
+      </xsl:when>
+      <xsl:otherwise>
+        <xsl:apply-templates />
+      </xsl:otherwise>
+    </xsl:choose>
+    <!-- p and div tags must end with a newline -->
+    <xsl:text>
+</xsl:text>
+  </xsl:template>
+
+  <xsl:template match="table">
+    <xsl:text>{| </xsl:text>
+
+    <!-- attributes -->
+    <xsl:for-each select="@*[local-name()!='id']">
+      <xsl:value-of select="local-name(.)" />
+      <xsl:text>="</xsl:text>
+      <xsl:value-of select="." />
+      <xsl:text>" </xsl:text>
+    </xsl:for-each>
+    <xsl:text>
+|-</xsl:text>
+
+    <xsl:apply-templates />
+    <xsl:text>
+|}
+
+</xsl:text>
+  </xsl:template>
+
+  <xsl:template match="tr">
+    <xsl:apply-templates />
+    <xsl:text>
+|-</xsl:text>
+  </xsl:template>
+
+  <xsl:template match="th|td">
+    <xsl:choose>
+      <xsl:when test="local-name(.) = 'th'">
+        <xsl:text>
+! </xsl:text>
+      </xsl:when>
+      <xsl:when test="local-name(.) = 'td'">
+        <xsl:text>
+| </xsl:text>
+      </xsl:when>
+    </xsl:choose>
+
+    <!-- attributes -->
+    <xsl:if test="@*">
+      <xsl:for-each select="@*">
+        <xsl:value-of select="local-name(.)" />
+        <xsl:text>="</xsl:text>
+        <xsl:value-of select="." />
+        <xsl:text>" </xsl:text>
+      </xsl:for-each>
+      <xsl:text>| </xsl:text>
+    </xsl:if>
+
+    <xsl:apply-templates />
+  </xsl:template>
+
+  <!-- don't end p tags with newlines in tables -->
+  <xsl:template match="table//p">
+    <xsl:apply-templates />
+  </xsl:template>
+
+  <!-- END tables -->
+
+  <xsl:template match="a[href]">
+    <xsl:text>[</xsl:text>
+    <xsl:apply-templates />
+    <xsl:text>](</xsl:text>
+    <xsl:value-of select="@href" />
+    <xsl:text>)</xsl:text>
+  </xsl:template>
+
+  <xsl:template match="img">
+    <xsl:text>![</xsl:text>
+    <xsl:value-of select="@alt" />
+    <xsl:text>](</xsl:text>
+    <xsl:value-of select="@src" />
+    <xsl:text>)</xsl:text>
+  </xsl:template>
+
+  <xsl:template match="br">
+    <xsl:text>
+</xsl:text>
+  </xsl:template>
+
+
+  <!-- for most nodes, just dump their text content -->
+  <xsl:template match="*">
+    <xsl:text/><xsl:apply-templates /><xsl:text/>
+  </xsl:template>
+  
+</xsl:stylesheet>

data/lib/slaw/extract/yomu_patch.rb

@@ -2,7 +2,7 @@ require 'yomu'
 
 class Yomu
   def self.text_from_file(filename)
-    IO.popen("#{java} -Djava.awt.headless=true -jar #{Yomu::JARPATH} -t '#{filename}'", 'r') do |io|
+    IO.popen("#{java} -Djava.awt.headless=true -jar #{Yomu::JARPATH} --html '#{filename}'", 'r') do |io|
       io.read
     end
   end

data/lib/slaw/version.rb

@@ -1,3 +1,3 @@
 module Slaw
-  VERSION = "0.14.2"
+  VERSION = "0.15.0"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: slaw
 version: !ruby/object:Gem::Version
-  version: 0.14.2
+  version: 0.15.0
 platform: ruby
 authors:
 - Greg Kempe
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2017-12-11 00:00:00.000000000 Z
+date: 2017-12-22 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -185,6 +185,7 @@ files:
 - lib/slaw/bylaw.rb
 - lib/slaw/collection.rb
 - lib/slaw/extract/extractor.rb
+- lib/slaw/extract/html_to_akn_text.xsl
 - lib/slaw/extract/yomu_patch.rb
 - lib/slaw/generator.rb
 - lib/slaw/lifecycle_event.rb