pdftdx 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 87f8818e7d617b36b12f835b98aa2b93acf47a1d
+  data.tar.gz: 743387ff99d01c64d164c864ebef401802529c63
+SHA512:
+  metadata.gz: 0742c4e3b4ccc06926a1ff2a71721ed5c545078f54b67b0554364d8c405970ea45788c37f33c1d6426700099feedd4ab7bc8dd6a67f5383c136ffefc3ae59a2c
+  data.tar.gz: 4f85e5f1bcb20146fc1223afa80c6993e2fc379193e246bcf7b34c2422620ce9e8674e4262a55e9edfe2afb59cdc572685b351a091f64056450570e1b4ffe5d7

data/.gitignore

@@ -0,0 +1,10 @@
+/.bundle/
+/.yardoc
+/Gemfile.lock
+/_yardoc/
+/coverage/
+/doc/
+/pkg/
+/spec/reports/
+/tmp/
+.idea

data/.idea/.rakeTasks

@@ -0,0 +1,7 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<Settings><!--This file was automatically generated by Ruby plugin.
+You are allowed to: 
+1. Remove rake task
+2. Add existing rake tasks
+To add existing rake tasks automatically delete this file and reload the project.
+--><RakeGroup description="" fullCmd="" taksId="rake"><RakeTask description="Build pdftdx-0.1.0.gem into the pkg directory" fullCmd="build" taksId="build" /><RakeTask description="Remove any temporary products" fullCmd="clean" taksId="clean" /><RakeTask description="Remove any generated files" fullCmd="clobber" taksId="clobber" /><RakeTask description="Build and install pdftdx-0.1.0.gem into system gems" fullCmd="install" taksId="install" /><RakeGroup description="" fullCmd="" taksId="install"><RakeTask description="Build and install pdftdx-0.1.0.gem into system gems without network access" fullCmd="install:local" taksId="local" /></RakeGroup><RakeTask description="Create tag v0.1.0 and build and push pdftdx-0.1.0.gem to Rubygems" fullCmd="release[remote]" taksId="release[remote]" /><RakeTask description="" fullCmd="default" taksId="default" /><RakeTask description="" fullCmd="release" taksId="release" /><RakeGroup description="" fullCmd="" taksId="release"><RakeTask description="" fullCmd="release:guard_clean" taksId="guard_clean" /><RakeTask description="" fullCmd="release:rubygem_push" taksId="rubygem_push" /><RakeTask description="" fullCmd="release:source_control_push" taksId="source_control_push" /></RakeGroup></RakeGroup></Settings>

data/.idea/inspectionProfiles/Project_Default.xml

@@ -0,0 +1,10 @@
+<component name="InspectionProjectProfileManager">
+  <profile version="1.0">
+    <option name="myName" value="Project Default" />
+    <inspection_tool class="SpellCheckingInspection" enabled="false" level="TYPO" enabled_by_default="false">
+      <option name="processCode" value="true" />
+      <option name="processLiterals" value="true" />
+      <option name="processComments" value="true" />
+    </inspection_tool>
+  </profile>
+</component>

data/.idea/inspectionProfiles/profiles_settings.xml

@@ -0,0 +1,7 @@
+<component name="InspectionProjectProfileManager">
+  <settings>
+    <option name="PROJECT_PROFILE" value="Project Default" />
+    <option name="USE_PROJECT_PROFILE" value="true" />
+    <version value="1.0" />
+  </settings>
+</component>

data/.idea/misc.xml

@@ -0,0 +1,14 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project version="4">
+  <component name="ProjectLevelVcsManager" settingsEditedManually="false">
+    <OptionsSetting value="true" id="Add" />
+    <OptionsSetting value="true" id="Remove" />
+    <OptionsSetting value="true" id="Checkout" />
+    <OptionsSetting value="true" id="Update" />
+    <OptionsSetting value="true" id="Status" />
+    <OptionsSetting value="true" id="Edit" />
+    <ConfirmationsSetting value="0" id="Add" />
+    <ConfirmationsSetting value="0" id="Remove" />
+  </component>
+  <component name="ProjectRootManager" version="2" project-jdk-name="RVM: ruby-2.3.0 [global]" project-jdk-type="RUBY_SDK" />
+</project>

data/.idea/modules.xml

@@ -0,0 +1,8 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project version="4">
+  <component name="ProjectModuleManager">
+    <modules>
+      <module fileurl="file://$PROJECT_DIR$/.idea/pdftdx.iml" filepath="$PROJECT_DIR$/.idea/pdftdx.iml" />
+    </modules>
+  </component>
+</project>

data/.idea/pdftdx.iml

@@ -0,0 +1,11 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<module type="RUBY_MODULE" version="4">
+  <component name="NewModuleRootManager">
+    <content url="file://$MODULE_DIR$" />
+    <orderEntry type="inheritedJdk" />
+    <orderEntry type="sourceFolder" forTests="false" />
+    <orderEntry type="library" scope="PROVIDED" name="bundler (v1.12.5, RVM: ruby-2.3.0 [global]) [gem]" level="application" />
+    <orderEntry type="library" scope="PROVIDED" name="pdftohtml (v0.2.1, RVM: ruby-2.3.0 [global]) [gem]" level="application" />
+    <orderEntry type="library" scope="PROVIDED" name="rake (v10.5.0, RVM: ruby-2.3.0 [global]) [gem]" level="application" />
+  </component>
+</module>

data/.idea/vcs.xml

@@ -0,0 +1,6 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project version="4">
+  <component name="VcsDirectoryMappings">
+    <mapping directory="$PROJECT_DIR$" vcs="Git" />
+  </component>
+</project>

data/Gemfile

@@ -0,0 +1,2 @@
+source 'https://rubygems.org'
+gemspec

data/Gemfile.lock

@@ -0,0 +1,22 @@
+PATH
+  remote: .
+  specs:
+    pdftdx (0.1.0)
+      pdftohtml
+
+GEM
+  remote: https://rubygems.org/
+  specs:
+    pdftohtml (0.2.1)
+    rake (10.5.0)
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  bundler (~> 1.12)
+  pdftdx!
+  rake (~> 10.0)
+
+BUNDLED WITH
+   1.12.5

data/LICENSE.txt

@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright (c) 2016 Paul Duncan
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

data/README.md

@@ -0,0 +1,7 @@
+# PDF Table Data Extractor
+
+Simple tool to extract Table Data from PDFs
+
+## License
+
+The gem is available as open source under the terms of the [MIT License](http://opensource.org/licenses/MIT).

data/Rakefile

@@ -0,0 +1,2 @@
+require "bundler/gem_tasks"
+task :default => :spec

data/bin/pdftdx

@@ -0,0 +1,26 @@
+#!/usr/bin/env ruby
+
+# PDF Table Data Extractor
+# by Eresse <eresse@eresse.net>
+
+# Internal Includes
+require 'pdftdx'
+
+# Usage
+def usage
+	puts "Usage: #{$0} <PDF_FILE>"
+	exit 1
+end
+
+# Main
+def main args
+
+	# Check Args
+	usage unless args.length == 1
+
+	# Extract Data from provided PDF File\
+	PDFTDX.extract_data args[0]
+end
+
+# Call Main
+main ARGV

data/lib/pdftdx/parser.rb

@@ -0,0 +1,80 @@
+# PDF Table Data Extractor
+# by Eresse <eresse@eresse.net>
+
+# External Includes
+require 'htmlentities'
+
+# Internal Includes
+require 'pdftdx/version'
+
+# PDF TDX Module
+module PDFTDX
+
+	# Parser Module
+	module Parser
+
+		# Line Regex
+		LINE_REGEX = /^<p style[^>]+top:([0-9]+)px[^>]+left:([0-9]+)px[^>]+>(.*)<\/p>/
+
+		# Maximum Cell Length (to be considered usable data)
+		MAX_CELL_LEN = 100
+
+		# Page Offset
+		PAGE_OFF = 10000
+
+		# Title Cell Regex
+		TITLE_CELL_REGEX = /<bbb>/
+
+		# Check Same Line
+		def self.same_line data, idx_a, idx_b
+			data[idx_a][:top] == data[idx_b][:top]
+		end
+
+		# Is All Same Data
+		def self.is_all_same row_data
+			n = row_data[row_data.keys[0]]
+			row_data.inject(true) { |b, e| b && (e[1] == n) }
+		end
+
+		# Contains Unusable Data (Empty / Long Strings)
+		def self.contains_unusable row_data
+			row_data.inject(false) { |b, e| b || (e[1].length == 0) || (e[1].length > MAX_CELL_LEN) }
+		end
+
+		# Process Data
+		def self.process_data data
+
+			# Build Data Table
+			table = {}
+			data.each { |d| table[d[:top]] ||= {}; table[d[:top]][d[:left]] = d[:data] }
+
+			# Filter Table Rows (Remove Lone Elements & Footers)
+			table.reject! { |top, row| row.size < 2 || (top % PAGE_OFF) >= 1110 || is_all_same(row) || contains_unusable(row) }
+
+			# Filter Table Cells
+			table = table.collect { |_top, r| r.reject { |_left, d| TITLE_CELL_REGEX =~ d } }.reject { |r| r.size < 1 }
+
+			# Cleanup Table ( IS THIS NECESSARY ? )
+			table.reject! { |r| r.size < 2 }
+
+			# DEBUG
+			puts "=============> #{table}"
+		end
+
+		# HTML Filter
+		def self.hfilter s
+			s.gsub '<br/>', "\n"
+		end
+
+		# Process Page Files
+		def self.process_page_files page_data
+
+			# Build HTML Entity Decoder
+			coder = HTMLEntities.new
+
+			# Collect & Process File Data
+			off = 0
+			process page_data.collect { |_idx, page| off = off + PAGE_OFF; page.select { |l| LINE_REGEX =~ l }.collect { |l| LINE_REGEX.match l }.collect { |d| { top: off + d[1].to_i, left: d[2].to_i, data: hfilter(coder.decode(d[3])) } } }.flatten
+		end
+	end
+end

data/lib/pdftdx/version.rb

@@ -0,0 +1,9 @@
+# PDF Table Data Extractor
+# by Eresse <eresse@eresse.net>
+
+# PDF TDX Module
+module PDFTDX
+
+	# Version
+	VERSION = '0.1.0'
+end

data/lib/pdftdx.rb

@@ -0,0 +1,25 @@
+# PDF Table Data Extractor
+# by Eresse <eresse@eresse.net>
+
+# External Includes
+require 'htmlentities'
+require 'pdftohtml'
+
+# Internal Includes
+require 'pdftdx/dumper'
+require 'pdftdx/parser'
+require 'pdftdx/version'
+
+# PDF TDX Module
+module PDFTDX
+
+	# Extract Data from PDF
+	def self.extract_data pdf_file
+
+		# Dump PDF Data
+		page_data = Pdftohtml.convert pdf_file
+
+		# Process Page Data
+		PDFTDX::Parser.process_page_files page_data
+	end
+end

data/pdftdx.gemspec

@@ -0,0 +1,25 @@
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'pdftdx/version'
+
+Gem::Specification.new do |spec|
+	spec.name = "pdftdx"
+	spec.version = PDFTDX::VERSION
+	spec.authors = ["Eresse"]
+	spec.email = ["eresse@eresse.net"]
+
+	spec.summary = "Simple PDF Table Data Extractor"
+	spec.description = "Simple tool to extract Table Data from PDFs"
+	spec.homepage = "http://redmine.eresse.net/projects/pdftdx"
+	spec.license = "MIT"
+
+	spec.files = `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features)/}) }
+	spec.bindir = "exe"
+	spec.executables = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
+	spec.require_paths = ["lib"]
+
+	spec.add_development_dependency "bundler", "~> 1.12"
+	spec.add_development_dependency "rake", "~> 10.0"
+	spec.add_runtime_dependency "pdftohtml"
+end

metadata

@@ -0,0 +1,104 @@
+--- !ruby/object:Gem::Specification
+name: pdftdx
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- Eresse
+autorequire: 
+bindir: exe
+cert_chain: []
+date: 2016-12-09 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.12'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.12'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+- !ruby/object:Gem::Dependency
+  name: pdftohtml
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+description: Simple tool to extract Table Data from PDFs
+email:
+- eresse@eresse.net
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- ".idea/.rakeTasks"
+- ".idea/inspectionProfiles/Project_Default.xml"
+- ".idea/inspectionProfiles/profiles_settings.xml"
+- ".idea/misc.xml"
+- ".idea/modules.xml"
+- ".idea/pdftdx.iml"
+- ".idea/vcs.xml"
+- Gemfile
+- Gemfile.lock
+- LICENSE.txt
+- README.md
+- Rakefile
+- bin/pdftdx
+- lib/pdftdx.rb
+- lib/pdftdx/parser.rb
+- lib/pdftdx/version.rb
+- pdftdx.gemspec
+homepage: http://redmine.eresse.net/projects/pdftdx
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.5.1
+signing_key: 
+specification_version: 4
+summary: Simple PDF Table Data Extractor
+test_files: []