anystyle-cli 1.0.1 → 1.3.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: db54794068efeafdbcb793c71ffeff40fa3e67f5ac7e4564eb76c2cddc23ec09
-  data.tar.gz: 4db1f6bf5a8529cd9dbf736745401a83ef207642c595c16857d903468488ddb6
+  metadata.gz: 25fa9822dfe3ab9862a84f5c31ae70df159643a3ecc40c4958ef0ed77b17da54
+  data.tar.gz: cf7587eec67a7372df802f4a7ee63e4ef1bc915070137c56d424cb2fcb1e813d
 SHA512:
-  metadata.gz: bda0fd4dd07c77a189a8964e4ec150d63a2ff80b324f5ea3e28188e6c1ce2947ca007df3fc5142dc84bc05b17a0cb0cfc960908c78532ec8723f940521ee900a
-  data.tar.gz: baecc0a27805e48906a23f9f08ddf1af1e22c83015993cef4624151778a5bdee3ad80bc50b6437d65f5a753c0f62f229463d8a8c162ff0faec44f4d6f206dbe9
+  metadata.gz: f1a0b7ec3479c88d429fc1b58855473fefc0cdfac9171a72c2124e3fb2e75b3503caf9963d072c74da5a55673af69ca1db9cc751dcf1ece6ae58d997a5837700
+  data.tar.gz: c8c410d03fe9dc43a1667d77f0931fd96d4f004ab9a1d45b21bf4eacad68054f3a301ea0241a42a42affd77dda7fbef24f042e1d3a4e4dc702da4b261a306dd5

data/README.md

@@ -10,22 +10,28 @@ anystyle --help
         anystyle [global options] command [command options] [arguments...]
 
     VERSION
-        1.0.0 (cli 1.0.0, data 1.2.0)
+        1.1.0 (cli 1.0.2, data 1.2.0)
 
     GLOBAL OPTIONS
-        --adapter=name       - Set the dictionary adapter (default: ruby)
-        -f, --format=name    - Set the output format (default: ["json"])
-        --help               - Show this message
-        --[no-]stdout        - Print results directly to stdout
-        --[no-]verbose       - Print status messages to stderr
-        --version            - Display the program version
-        -w, --[no-]overwrite - Allow overwriting existing files
+        -F, --finder-model=file - Set the finder model file (default: none)
+        -P, --parser-model=file - Set the parser model file (default: none)
+        --adapter=name          - Set the dictionary adapter (default: ruby)
+        -f, --format=name       - Set the output format (default: ["json"])
+        --pdfinfo=path          - Set the path for pdfinfo (default: none)
+        --pdftotext=path        - Set the path for pdftotext (default: none)
+        --help                  - Show this message
+        --[no-]stdout           - Print results directly to stdout
+        --[no-]verbose          - Print status messages to stderr
+        --version               - Display the program version
+        -w, --[no-]overwrite    - Allow overwriting existing files
 
     COMMANDS
+        check   - Check tagged documents or references
         find    - Find and extract references from text documents
         help    - Shows a list of commands or help for one command
         license - Print license information
         parse   - Parse and convert references
+        train   - Create a new finder or parser model
 
 anystyle help find
 ------------------
@@ -57,7 +63,6 @@ anystyle help find
 
         Anlyzing PDF documents currently depends on `pdftotext' which must be
         installed separately.
-
     EXAMPLES
         anystyle -f csl,xml find thesis.pdf
 
@@ -70,6 +75,24 @@ anystyle help find
         if your document uses a multi-column layout) and save them in BibTeX in
         `./bib/thesis.bib'.
 
+        anystyle find --crop 72 thesis.pdf -
+
+        Extract references from `thesis.pdf' cropping away one inch (72pt) from
+        each page border and print the results to STDOUT.
+
+        anystyle find --crop 72,28 thesis.pdf -
+
+        Extract references from `thesis.pdf' cropping away one inch (72pt) from
+        each page's left and right border, approx. 1cm (28pt) from the top
+        and bottom.
+
+
+    COMMAND OPTIONS
+        -C, --crop=pt - Set cropping boundary for text extraction (default: none)
+        --[no-]layout - Use layout mode for PDF text extraction (default: enabled)
+        --[no-]solo   - Include references outside of reference sections
+
+
 anystyle help parse
 -------------------
     COMMAND OPTIONS
@@ -111,6 +134,55 @@ anystyle help parse
 
         Extract references from `input.txt' and print them to STDOUT in CSL/JSON.
 
+
+anystyle help check
+-------------------
+    NAME
+        check - Check tagged documents or references
+
+    SYNOPSIS
+        anystyle [global options] check input
+
+    DESCRIPTION
+        This manual page documents the AnyStyle `check' command. AnyStyle `check'
+        analyzes tagged text documents or references.
+
+        The input argument can be a single TTX or XML document, or a folder
+        containing multiple documents.
+
+        AnyStyle `check' supports the following input formats:
+            ttx     Tagged document format, used for training the finder model;
+            xml     References only, XML, suitable for training the parser model.
+
+    EXAMPLES
+        anystyle check training-data.xml
+
+        Checks all references in the XML file and prints a report to STDOUT.
+
+
+anystyle help train
+-------------------
+    NAME
+        train - Create a new finder or parser model
+
+    SYNOPSIS
+        anystyle [global options] train input [output]
+
+    DESCRIPTION
+        This manual page documents the AnyStyle `train' command. AnyStyle `train'
+        creates a new finder or parser model based on the supplied training sets.
+
+        The input argument can be a XML document, or a folder containing multiple
+        TTX documents.
+
+    EXAMPLES
+        anystyle train data.xml my-model.mod
+
+        Creates a new parser model based on the XML training set and saves it
+        as `my-model.mod'. To use your model use the global `--finder-model'
+        or `--parser-model' flags.
+
+
 License
 -------
 Copyright 2011-2018 Sylvester Keil. All rights reserved.

data/bin/anystyle

@@ -30,6 +30,23 @@ switch ['w', 'overwrite'],
 switch 'stdout',
   desc: 'Print results directly to stdout'
 
+flag ['F', 'finder-model'],
+  arg_name: 'file',
+  desc: 'Set the finder model file'
+
+flag ['P', 'parser-model'],
+  arg_name: 'file',
+  desc: 'Set the parser model file'
+
+flag 'pdftotext',
+  arg_name: 'path',
+  desc: 'Set the path for pdftotext'
+
+flag 'pdfinfo',
+  arg_name: 'path',
+  desc: 'Set the path for pdfinfo'
+
+
 flag 'adapter',
   default_value: 'ruby',
   arg_name: 'name',
@@ -46,6 +63,28 @@ flag ['f', 'format'],
 
 pre do |opts|
   AnyStyle::Dictionary.defaults[:adapter] = opts[:adapter]
+
+  unless opts[:'finder-model'].nil?
+    AnyStyle::Finder.defaults[:model] =
+      File.expand_path(opts[:'finder-model']).untaint
+  end
+
+  unless opts[:'parser-model'].nil?
+    AnyStyle::Parser.defaults[:model] =
+      File.expand_path(opts[:'parser-model']).untaint
+  end
+
+  unless opts[:pdftotext].nil?
+    AnyStyle::Finder.defaults[:pdftotext] =
+      opts[:pdftotext].untaint
+  end
+
+  unless opts[:pdfinfo].nil?
+    AnyStyle::Finder.defaults[:pdfinfo] =
+      opts[:pdfinfo].untaint
+  end
+
+  AnyStyle
 end
 
 
@@ -84,6 +123,17 @@ EXAMPLES
     Extract references from `thesis.pdf' in `no-layout' mode (e.g., use this
     if your document uses a multi-column layout) and save them in BibTeX in
     `./bib/thesis.bib'.
+
+    anystyle find --crop 72 thesis.pdf -
+
+    Extract references from `thesis.pdf' cropping away one inch (72pt) from
+    each page border and print the results to STDOUT.
+
+    anystyle find --crop 72,28 thesis.pdf -
+
+    Extract references from `thesis.pdf' cropping away one inch (72pt) from
+    each page's left and right border, approx. 1cm (28pt) from the top
+    and bottom.
 }.lstrip
 
 arg :input
@@ -93,6 +143,16 @@ command :find do |cmd|
     default_value: true,
     desc: 'Use layout mode for PDF text extraction'
 
+  cmd.switch 'solo',
+    default_value: false,
+    desc: 'Include references outside of reference sections'
+
+  cmd.flag ['C', 'crop'],
+    arg_name: 'pt',
+    type: Array,
+    must_match: /\d+(,\d+)?|\d+,\d+(,-?\d+){2}/,
+    desc: 'Set cropping boundary for text extraction'
+
   cmd.action do |opts, params, args|
     Commands::Find.new(opts).run(args, params)
   end
@@ -140,6 +200,56 @@ command :parse do |cmd|
   end
 end
 
+desc 'Check tagged documents or references'
+long_desc %{
+    This manual page documents the AnyStyle `check' command. AnyStyle `check'
+    analyzes tagged text documents or references.
+
+    The input argument can be a single TTX or XML document, or a folder
+    containing multiple documents.
+
+    AnyStyle `check' supports the following input formats:
+        ttx     Tagged document format, used for training the finder model;
+        xml     References only, XML, suitable for training the parser model.
+
+EXAMPLES
+    anystyle check training-data.xml
+
+    Checks all references in the XML file and prints a report to STDOUT.
+}.lstrip
+
+arg :input
+command :check do |cmd|
+  cmd.action do |opts, params, args|
+    Commands::Check.new(opts).run(args, params)
+  end
+end
+
+
+desc 'Create a new finder or parser model'
+long_desc %{
+    This manual page documents the AnyStyle `train' command. AnyStyle `train'
+    creates a new finder or parser model based on the supplied training sets.
+
+    The input argument can be a XML document, or a folder containing multiple
+    TTX documents.
+
+EXAMPLES
+    anystyle train data.xml my-model.mod
+
+    Creates a new parser model based on the XML training set and saves it
+    as `my-model.mod'. To use your model use the global `--finder-model'
+    or `--parser-model' flags.
+}.lstrip
+
+arg :input
+arg :output, :optional
+command :train do |cmd|
+  cmd.action do |opts, params, args|
+    Commands::Train.new(opts).run(args, params)
+  end
+end
+
 desc 'Print license information'
 command :license do |cmd|
   cmd.action do

data/lib/anystyle/cli.rb

@@ -4,5 +4,7 @@ require 'anystyle'
 
 require 'anystyle/cli/version'
 require 'anystyle/cli/commands/base'
+require 'anystyle/cli/commands/check'
 require 'anystyle/cli/commands/find'
 require 'anystyle/cli/commands/parse'
+require 'anystyle/cli/commands/train'

data/lib/anystyle/cli/commands/base.rb

@@ -28,8 +28,11 @@ module AnyStyle
           options[:format].each(&block)
         end
 
-        def find(input, **opts)
-          AnyStyle.find(input, format: :wapiti, **opts)
+        def find(input, opts = {})
+          AnyStyle.find(input,
+            format: :wapiti,
+            layout: opts[:layout],
+            crop: opts[:crop].nil? ? nil : opts[:crop].map(&:to_i))
         end
 
         def parse(input)
@@ -67,7 +70,12 @@ module AnyStyle
         end
 
         def set_output_folder(path)
-          @output_folder = Pathname.new(path).expand_path unless path.nil?
+          case path
+          when nil, '-'
+            options[:stdout] = true
+          else
+            @output_folder = Pathname.new(path).expand_path
+          end
         ensure
           unless @output_folder.nil?
             if @output_folder.exist?
@@ -83,16 +91,32 @@ module AnyStyle
           STDERR.puts(*args) if verbose?
         end
 
+        def report(error, file)
+          STDERR.puts "Error processing `#{file}'"
+          STDERR.puts "  #{error.message}"
+          STDERR.puts "  #{error.backtrace[0]}"
+          STDERR.puts "  #{error.backtrace[1]}"
+          STDERR.puts "  ..."
+        end
+
         def walk(input)
           path = Pathname(input).expand_path
           raise ArgumentError, "path does not exist: #{input}" unless path.exist?
 
           if path.directory?
             path.each_child do |file|
-              yield file, path unless file.directory?
+              begin
+                yield file, path unless file.directory?
+              rescue => e
+                report e, file.relative_path_from(path)
+              end
             end
           else
-            yield path, path.dirname
+            begin
+              yield path, path.dirname
+            rescue => e
+              report e, path.basename
+            end
           end
         end
 

data/lib/anystyle/cli/commands/check.rb

@@ -0,0 +1,41 @@
+module AnyStyle
+  module CLI
+    module Commands
+      class Check < Base
+        def run(args, params)
+          walk args[0] do |path|
+            print 'Checking %.25s' % "#{path.basename}....................."
+            start = Time.now
+            stats = check path
+            report stats, Time.now - start
+          end
+        end
+
+        def check(path)
+          case path.extname
+          when '.ttx'
+            AnyStyle.finder.check path.to_s.untaint
+          when '.xml'
+            AnyStyle.parser.check path.to_s.untaint
+          else
+            raise ArgumentError, "cannot check untagged input: #{path}"
+          end
+        end
+
+        def report(stats, time)
+          if stats[:token][:errors] == 0
+            puts '   ✓                               %2ds' % time
+          else
+            puts '%4d seq %6.2f%% %6d tok %5.2f%% %2ds' % [
+              stats[:sequence][:errors],
+              stats[:sequence][:rate],
+              stats[:token][:errors],
+              stats[:token][:rate],
+              time
+            ]
+          end
+        end
+      end
+    end
+  end
+end

data/lib/anystyle/cli/commands/find.rb

@@ -6,8 +6,8 @@ module AnyStyle
           set_output_folder args[1]
           walk args[0] do |path, base_path|
             say "Analyzing #{path.relative_path_from(base_path)} ..."
-            doc = find(path.to_s.untaint, layout: params[:layout])
-            ref = doc[0].references
+            doc = find(path.to_s.untaint, params)
+            ref = doc[0].references(normalize_blocks: !params[:solo])
 
             if ref.length == 0
               say "no references found."
@@ -17,9 +17,9 @@ module AnyStyle
               each_format do |fmt|
                 case fmt
                 when 'ttx'
-                  res = doc.to_txt tagged: true
+                  res = doc.to_s tagged: true
                 when 'txt'
-                  res = doc.to_txt tagged: false
+                  res = doc.to_s tagged: false
                 when 'ref'
                   res = ref.join("\n")
                 else

data/lib/anystyle/cli/commands/train.rb

@@ -0,0 +1,41 @@
+module AnyStyle
+  module CLI
+    module Commands
+      class Train < Base
+        def run(args, params)
+          check_no_overwrite! args[1]
+
+          Wapiti.debug!
+          model = train(args[0])
+
+          if args[1].nil?
+            model.save
+          else
+            model.save File.expand_path(args[1]).untaint
+          end
+        end
+
+        def train(path)
+          case
+          when File.extname(path) == '.xml'
+            AnyStyle.parser.train path.to_s.untaint
+            AnyStyle.parser.model
+          when File.directory?(path)
+            AnyStyle.finder.train Dir[File.join(path, '*.ttx')].map(&:untaint)
+            AnyStyle.finder.model
+          else
+            raise ArgumentError, "cannot train input: #{path}"
+          end
+        end
+
+        def check_no_overwrite!(path)
+          if !overwrite? && (path.nil? || File.exist?(path))
+            raise RuntimeError,
+              "file exists, use --overwrite to force saving: #{path}"
+          end
+        end
+      end
+    end
+  end
+end
+

data/lib/anystyle/cli/version.rb

@@ -1,5 +1,5 @@
 module AnyStyle
   module CLI
-    VERSION = '1.0.1'.freeze
+    VERSION = '1.3.1'.freeze
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: anystyle-cli
 version: !ruby/object:Gem::Version
-  version: 1.0.1
+  version: 1.3.1
 platform: ruby
 authors:
 - Sylvester Keil
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2018-06-06 00:00:00.000000000 Z
+date: 2020-05-26 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: anystyle
@@ -16,28 +16,28 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.0'
+        version: '1.3'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.0'
+        version: '1.3'
 - !ruby/object:Gem::Dependency
   name: bibtex-ruby
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '4.4'
+        version: '5.1'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '4.4'
+        version: '5.1'
 - !ruby/object:Gem::Dependency
   name: gli
   requirement: !ruby/object:Gem::Requirement
@@ -65,8 +65,10 @@ files:
 - bin/anystyle
 - lib/anystyle/cli.rb
 - lib/anystyle/cli/commands/base.rb
+- lib/anystyle/cli/commands/check.rb
 - lib/anystyle/cli/commands/find.rb
 - lib/anystyle/cli/commands/parse.rb
+- lib/anystyle/cli/commands/train.rb
 - lib/anystyle/cli/version.rb
 homepage: http://anystyle.io
 licenses:
@@ -87,8 +89,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubyforge_project: 
-rubygems_version: 2.7.4
+rubygems_version: 3.1.2
 signing_key: 
 specification_version: 4
 summary: AnyStyle CLI