anystyle-cli 1.0.2 → 1.1.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
-SHA256:
-  metadata.gz: '082b162dc4771fef9ba5ec4ee4fa623931ba0e34c8617a5b47ca7545bb14ce58'
-  data.tar.gz: d035793b06dc909b570defd917995a782fe5cee3eafc9674aa3e696b64e73e98
+SHA1:
+  metadata.gz: c6c4e6580d9495afd965e3401f030cee94925747
+  data.tar.gz: 56e3842f3b3db05ea1a7a5b17ab4f72b5ff07773
 SHA512:
-  metadata.gz: a2836a714f4071917502ee641733d491c33710fa9b0e07f113ebf1c07c3bf2f2a97f9d5621312ca2eb942768a5b481597e03573ffa27cdc31ecba5d02351a7eb
-  data.tar.gz: d51da26bceb9b7f2dd56ae1322145035c8669ad52ab5e0ac7bb11a6f9972ad3f2a757cfbb3e61272855d5605be4b3de40a44c1c2a644e1bea744e9bcd0fb319f
+  metadata.gz: 01b90daac809b52ac69e70550952596512ef8df0bdc5503faabf4a173e83d4bf045dd2b0fcf4ae0bb28b165b182885e6f997f5f6f764638ed7b19dda67a1a87d
+  data.tar.gz: be29161a8a6ba925f2f21a4d50ad1cb41c93d945977cc3cdf026b58383e3c1d0f979c44966cb32934c1ce2d24d0bbba7751e8b9f771927c9c43d2eb6b8385546

data/README.md

@@ -10,22 +10,26 @@ anystyle --help
         anystyle [global options] command [command options] [arguments...]
 
     VERSION
-        1.0.0 (cli 1.0.0, data 1.2.0)
+        1.1.0 (cli 1.0.2, data 1.2.0)
 
     GLOBAL OPTIONS
-        --adapter=name       - Set the dictionary adapter (default: ruby)
-        -f, --format=name    - Set the output format (default: ["json"])
-        --help               - Show this message
-        --[no-]stdout        - Print results directly to stdout
-        --[no-]verbose       - Print status messages to stderr
-        --version            - Display the program version
-        -w, --[no-]overwrite - Allow overwriting existing files
+        -F, --finder-model=file - Set the finder model file (default: none)
+        -P, --parser-model=file - Set the parser model file (default: none)
+        --adapter=name          - Set the dictionary adapter (default: ruby)
+        -f, --format=name       - Set the output format (default: ["json"])
+        --help                  - Show this message
+        --[no-]stdout           - Print results directly to stdout
+        --[no-]verbose          - Print status messages to stderr
+        --version               - Display the program version
+        -w, --[no-]overwrite    - Allow overwriting existing files
 
     COMMANDS
+        check   - Check tagged documents or references
         find    - Find and extract references from text documents
         help    - Shows a list of commands or help for one command
         license - Print license information
         parse   - Parse and convert references
+        train   - Create a new finder or parser model
 
 anystyle help find
 ------------------
@@ -57,7 +61,6 @@ anystyle help find
 
         Anlyzing PDF documents currently depends on `pdftotext' which must be
         installed separately.
-
     EXAMPLES
         anystyle -f csl,xml find thesis.pdf
 
@@ -70,6 +73,24 @@ anystyle help find
         if your document uses a multi-column layout) and save them in BibTeX in
         `./bib/thesis.bib'.
 
+        anystyle find --crop 72 thesis.pdf -
+
+        Extract references from `thesis.pdf' cropping away one inch (72pt) from
+        each page border and print the results to STDOUT.
+
+        anystyle find --crop 72,28 thesis.pdf -
+
+        Extract references from `thesis.pdf' cropping away one inch (72pt) from
+        each page's left and right border, approx. 1cm (28pt) from the top
+        and bottom.
+
+
+    COMMAND OPTIONS
+        -C, --crop=pt - Set cropping boundary for text extraction (default: none)
+        --[no-]layout - Use layout mode for PDF text extraction (default: enabled)
+        --[no-]solo   - Include references outside of reference sections
+
+
 anystyle help parse
 -------------------
     COMMAND OPTIONS
@@ -111,6 +132,55 @@ anystyle help parse
 
         Extract references from `input.txt' and print them to STDOUT in CSL/JSON.
 
+
+anystyle help check
+-------------------
+    NAME
+        check - Check tagged documents or references
+
+    SYNOPSIS
+        anystyle [global options] check input
+
+    DESCRIPTION
+        This manual page documents the AnyStyle `check' command. AnyStyle `check'
+        analyzes tagged text documents or references.
+
+        The input argument can be a single TTX or XML document, or a folder
+        containing multiple documents.
+
+        AnyStyle `check' supports the following input formats:
+            ttx     Tagged document format, used for training the finder model;
+            xml     References only, XML, suitable for training the parser model.
+
+    EXAMPLES
+        anystyle check training-data.xml
+
+        Checks all references in the XML file and prints a report to STDOUT.
+
+
+anystyle help train
+-------------------
+    NAME
+        train - Create a new finder or parser model
+
+    SYNOPSIS
+        anystyle [global options] train input [output]
+
+    DESCRIPTION
+        This manual page documents the AnyStyle `train' command. AnyStyle `train'
+        creates a new finder or parser model based on the supplied training sets.
+
+        The input argument can be a XML document, or a folder containing multiple
+        TTX documents.
+
+    EXAMPLES
+        anystyle train data.xml my-model.mod
+
+        Creates a new parser model based on the XML training set and saves it
+        as `my-model.mod'. To use your model use the global `--finder-model'
+        or `--parser-model' flags.
+
+
 License
 -------
 Copyright 2011-2018 Sylvester Keil. All rights reserved.

data/bin/anystyle

@@ -30,6 +30,14 @@ switch ['w', 'overwrite'],
 switch 'stdout',
   desc: 'Print results directly to stdout'
 
+flag ['F', 'finder-model'],
+  arg_name: 'file',
+  desc: 'Set the finder model file'
+
+flag ['P', 'parser-model'],
+  arg_name: 'file',
+  desc: 'Set the parser model file'
+
 flag 'adapter',
   default_value: 'ruby',
   arg_name: 'name',
@@ -46,6 +54,18 @@ flag ['f', 'format'],
 
 pre do |opts|
   AnyStyle::Dictionary.defaults[:adapter] = opts[:adapter]
+
+  unless opts[:'finder-model'].nil?
+    AnyStyle::Finder.defaults[:model] =
+      File.expand_path(opts[:'finder-model']).untaint
+  end
+
+  unless opts[:'parser-model'].nil?
+    AnyStyle::Parser.defaults[:model] =
+      File.expand_path(opts[:'parser-model']).untaint
+  end
+
+  AnyStyle
 end
 
 
@@ -84,6 +104,17 @@ EXAMPLES
     Extract references from `thesis.pdf' in `no-layout' mode (e.g., use this
     if your document uses a multi-column layout) and save them in BibTeX in
     `./bib/thesis.bib'.
+
+    anystyle find --crop 72 thesis.pdf -
+
+    Extract references from `thesis.pdf' cropping away one inch (72pt) from
+    each page border and print the results to STDOUT.
+
+    anystyle find --crop 72,28 thesis.pdf -
+
+    Extract references from `thesis.pdf' cropping away one inch (72pt) from
+    each page's left and right border, approx. 1cm (28pt) from the top
+    and bottom.
 }.lstrip
 
 arg :input
@@ -93,6 +124,16 @@ command :find do |cmd|
     default_value: true,
     desc: 'Use layout mode for PDF text extraction'
 
+  cmd.switch 'solo',
+    default_value: false,
+    desc: 'Include references outside of reference sections'
+
+  cmd.flag ['C', 'crop'],
+    arg_name: 'pt',
+    type: Array,
+    must_match: /\d+(,\d+)?|\d+,\d+(,-?\d+){2}/,
+    desc: 'Set cropping boundary for text extraction'
+
   cmd.action do |opts, params, args|
     Commands::Find.new(opts).run(args, params)
   end
@@ -140,6 +181,56 @@ command :parse do |cmd|
   end
 end
 
+desc 'Check tagged documents or references'
+long_desc %{
+    This manual page documents the AnyStyle `check' command. AnyStyle `check'
+    analyzes tagged text documents or references.
+
+    The input argument can be a single TTX or XML document, or a folder
+    containing multiple documents.
+
+    AnyStyle `check' supports the following input formats:
+        ttx     Tagged document format, used for training the finder model;
+        xml     References only, XML, suitable for training the parser model.
+
+EXAMPLES
+    anystyle check training-data.xml
+
+    Checks all references in the XML file and prints a report to STDOUT.
+}.lstrip
+
+arg :input
+command :check do |cmd|
+  cmd.action do |opts, params, args|
+    Commands::Check.new(opts).run(args, params)
+  end
+end
+
+
+desc 'Create a new finder or parser model'
+long_desc %{
+    This manual page documents the AnyStyle `train' command. AnyStyle `train'
+    creates a new finder or parser model based on the supplied training sets.
+
+    The input argument can be a XML document, or a folder containing multiple
+    TTX documents.
+
+EXAMPLES
+    anystyle train data.xml my-model.mod
+
+    Creates a new parser model based on the XML training set and saves it
+    as `my-model.mod'. To use your model use the global `--finder-model'
+    or `--parser-model' flags.
+}.lstrip
+
+arg :input
+arg :output, :optional
+command :train do |cmd|
+  cmd.action do |opts, params, args|
+    Commands::Train.new(opts).run(args, params)
+  end
+end
+
 desc 'Print license information'
 command :license do |cmd|
   cmd.action do

data/lib/anystyle/cli.rb

@@ -4,5 +4,7 @@ require 'anystyle'
 
 require 'anystyle/cli/version'
 require 'anystyle/cli/commands/base'
+require 'anystyle/cli/commands/check'
 require 'anystyle/cli/commands/find'
 require 'anystyle/cli/commands/parse'
+require 'anystyle/cli/commands/train'

data/lib/anystyle/cli/commands/base.rb

@@ -28,8 +28,11 @@ module AnyStyle
           options[:format].each(&block)
         end
 
-        def find(input, **opts)
-          AnyStyle.find(input, format: :wapiti, **opts)
+        def find(input, opts = {})
+          AnyStyle.find(input,
+            format: :wapiti,
+            layout: opts[:layout],
+            crop: opts[:crop].nil? ? nil : opts[:crop].map(&:to_i))
         end
 
         def parse(input)
@@ -67,7 +70,12 @@ module AnyStyle
         end
 
         def set_output_folder(path)
-          @output_folder = Pathname.new(path).expand_path unless path.nil?
+          case path
+          when nil, '-'
+            options[:stdout] = true
+          else
+            @output_folder = Pathname.new(path).expand_path
+          end
         ensure
           unless @output_folder.nil?
             if @output_folder.exist?

data/lib/anystyle/cli/commands/check.rb

@@ -0,0 +1,41 @@
+module AnyStyle
+  module CLI
+    module Commands
+      class Check < Base
+        def run(args, params)
+          walk args[0] do |path|
+            print 'Checking %.25s' % "#{path.basename}....................."
+            start = Time.now
+            stats = check path
+            report stats, Time.now - start
+          end
+        end
+
+        def check(path)
+          case path.extname
+          when '.ttx'
+            AnyStyle.finder.check path.to_s.untaint
+          when '.xml'
+            AnyStyle.parser.check path.to_s.untaint
+          else
+            raise ArgumentError, "cannot check untagged input: #{path}"
+          end
+        end
+
+        def report(stats, time)
+          if stats[:token][:errors] == 0
+            puts '   ✓                               %2ds' % time
+          else
+            puts '%4d seq %6.2f%% %6d tok %5.2f%% %2ds' % [
+              stats[:sequence][:errors],
+              stats[:sequence][:rate],
+              stats[:token][:errors],
+              stats[:token][:rate],
+              time
+            ]
+          end
+        end
+      end
+    end
+  end
+end

data/lib/anystyle/cli/commands/find.rb

@@ -6,8 +6,8 @@ module AnyStyle
           set_output_folder args[1]
           walk args[0] do |path, base_path|
             say "Analyzing #{path.relative_path_from(base_path)} ..."
-            doc = find(path.to_s.untaint, layout: params[:layout])
-            ref = doc[0].references
+            doc = find(path.to_s.untaint, params)
+            ref = doc[0].references(normalize_blocks: !params[:solo])
 
             if ref.length == 0
               say "no references found."

data/lib/anystyle/cli/commands/train.rb

@@ -0,0 +1,41 @@
+module AnyStyle
+  module CLI
+    module Commands
+      class Train < Base
+        def run(args, params)
+          check_no_overwrite! args[1]
+
+          Wapiti.debug!
+          model = train(args[0])
+
+          if args[1].nil?
+            model.save
+          else
+            model.save File.expand_path(args[1]).untaint
+          end
+        end
+
+        def train(path)
+          case
+          when File.extname(path) == '.xml'
+            AnyStyle.parser.train path.to_s.untaint
+            AnyStyle.parser.model
+          when File.directory?(path)
+            AnyStyle.finder.train path.to_s.untaint
+            AnyStyle.finder.model
+          else
+            raise ArgumentError, "cannot train input: #{path}"
+          end
+        end
+
+        def check_no_overwrite!(path)
+          if !overwrite? && (path.nil? || File.exist?(path))
+            raise RuntimeError,
+              "file exists, use --overwrite to force saving: #{path}"
+          end
+        end
+      end
+    end
+  end
+end
+

data/lib/anystyle/cli/version.rb

@@ -1,5 +1,5 @@
 module AnyStyle
   module CLI
-    VERSION = '1.0.2'.freeze
+    VERSION = '1.1.0'.freeze
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: anystyle-cli
 version: !ruby/object:Gem::Version
-  version: 1.0.2
+  version: 1.1.0
 platform: ruby
 authors:
 - Sylvester Keil
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2018-07-11 00:00:00.000000000 Z
+date: 2018-08-16 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: anystyle
@@ -16,14 +16,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.0'
+        version: '1.2'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.0'
+        version: '1.2'
 - !ruby/object:Gem::Dependency
   name: bibtex-ruby
   requirement: !ruby/object:Gem::Requirement
@@ -65,8 +65,10 @@ files:
 - bin/anystyle
 - lib/anystyle/cli.rb
 - lib/anystyle/cli/commands/base.rb
+- lib/anystyle/cli/commands/check.rb
 - lib/anystyle/cli/commands/find.rb
 - lib/anystyle/cli/commands/parse.rb
+- lib/anystyle/cli/commands/train.rb
 - lib/anystyle/cli/version.rb
 homepage: http://anystyle.io
 licenses:
@@ -88,7 +90,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.7.4
+rubygems_version: 2.6.13
 signing_key: 
 specification_version: 4
 summary: AnyStyle CLI