plain_text 0.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: 52d4c007bf2d127b5fed9c1edd5df87597ee5ca4689818244ecb05bcb6d0a8f1
+  data.tar.gz: 5af8e4489d714e272c8304911cbfdac18a736a49abf35d5143fd86ddf1d7e917
+SHA512:
+  metadata.gz: 422f9de75686466409ce8d9c819226313974d6d308aa66dbede7b808aed8230b5efb3bbf838151d00031a9157e9109c936e3af3bc2dc51e447bce9fee2bfd81d
+  data.tar.gz: af8ca2904b51fb3ea3b822a49ae3e650ef5112af8f63d4b1df51ade1b31a0d083f712bf2b4f64f5eba17624c11264e6b7ffc8de8ce866541ef8d47a160be299c

data/.gitignore

@@ -0,0 +1,51 @@
+# See https://help.github.com/articles/ignoring-files for more about ignoring files.
+#
+# If you find yourself ignoring temporary files generated by your text editor
+# or operating system, you probably want to add a global ignore instead:
+#   git config --global core.excludesfile '~/.gitignore_global'
+
+# Ignore bundler config.
+/.bundle
+/vendor/bundle
+
+# Ignore all logfiles and tempfiles.
+/log/*
+/tmp/*
+!/log/.keep
+!/tmp/.keep
+
+.rbenv-version
+# unless supporting rvm < 1.11.0 or doing something fancy, ignore this:
+.rvmrc
+
+/node_modules
+/yarn-error.log
+
+.byebug_history
+
+*.[oa]
+*.so
+*~
+*.nogem
+*nogem.*
+*.bak
+*.BAK
+*.backup
+*.org
+*.orig
+*.elc
+*.pyc
+\#*\#
+
+# Elastic Beanstalk Files
+.elasticbeanstalk/*
+!.elasticbeanstalk/*.cfg.yml
+!.elasticbeanstalk/*.global.yml
+
+# yard
+*.yardoc
+
+# Ruby Gem doc
+*.gem
+doc/*
+

data/ChangeLog

@@ -0,0 +1,5 @@
+-----
+(Version: 0.1)
+2019-10-25  Masa Sakano
+
+ * Initial preliminary commit.

data/Makefile

@@ -0,0 +1,23 @@
+ALL	= 
+
+objs	= 
+
+.SUFFIXES:	.so .o .c .f
+
+#.o.so:
+#	${LD} ${LFLAGS} -o $@ $< ${LINK_LIB}
+
+all: ${ALL}
+
+
+.PHONY: clean test doc
+clean:
+	$(RM) bin/*~
+
+## You may need RUBYLIB=`pwd`/lib:$RUBYLIB
+test:
+	rake test
+
+doc:
+	yard doc
+

data/README.en.rdoc

@@ -0,0 +1,172 @@
+
+= PlainText - Module and classes to handle plain text
+
+== Summary
+
+This module provides utility functions and methods to handle plain
+text.  In the namespace, classes Part/Paragraph/Boundary are defined,
+which represent the logical structure of a document and another class
+ParseRule, which describes the rules to parse plain text to produce a Part-type Ruby instance.
+This package also provides a command-line program to count the number
+of characters, especially useful for documents in Asian (CJK) chatacters.
+
+== Design concept
+
+=== PlainText - Module and root Namespace
+
+The original plain text should be in String in Ruby.
+
+The module {PlainText} offers some useful methods, such as,
+{PlainText#head} and {PlainText#tail}.  They are meant to be included
+in String.  However, it also contains some useful module functions,
+such as, {PlainText.clean_text} and {PlainText.count_char}.
+
+=== PlainText::Part - Core class to describe the logical structure
+
+In the namespace of this module, it contains {PlainText::Part} class,
+which is the heart to describe the logical structure of documents.
+It is basically a container class and indeed a sub-class of Array. It
+can contain either of another {PlainText::Part} or more basic
+components of either of {PlainText::Part::Paragraph} and
+{PlainText::Part::Boundary}, both of which are sub-classes of String. 
+
+An example instance looks like this:
+
+   Part (
+     (0) Paragraph::Empty,
+     (1) Boundary::General,
+     (2) Part::ArticleHeader(
+           (0) Paragraph::Title,
+           (1) Boundary::Empty
+         ),
+     (3) Boundary::TitleMain,
+     (4) Part::ArticleMain(
+           (0) Part::ArticleSection(
+                 (0) Paragraph::Title,
+                 (1) Boundary::General,
+                 (2) Paragraph::General,
+                 (3) Boundary::General,
+                 (4) Part::ArticleSubSection(...),
+                 (5) Boundary::General,
+                 (6) Paragraph::General,
+                 (7) Boundary::Empty
+               ),
+           (1) Boundary::General,
+           (2) Paragraph::General,
+           (3) Boundary::Empty
+         ),
+     (5) Boundary::General
+   )
+
+where the name of subclasses (or constants) here arbitrary, except for
+{PlainText::Part::Paragraph::Empty} and
+{PlainText::Part::Boundary::Empty}, which are pre-defined. Users can
+define their own subclasses to help organize the logical structure at
+their will. 
+
+Basically, at every layer, every {PlainText::Part} or
+{PlainText::Part::Paragraph} is sandwiched by
+{PlainText::Part::Boundary}, except for the very first one.
+
+By performing +join+ method, one can retrieve the entire document as a
+String instance any time.
+
+=== PlainText::ParseRule - Class to describe the rule of how to parse
+
+{PlainText::ParseRule} is the class to describe how to parse initially
+String, and subsequently {PlainText::Part}, which is basically an Array. 
+{PlainText::ParseRule} is a container class and holds a set of ordered
+rules, each of which is either Proc or Regexp as a more simple rule.
+A rule, Proc, is defined by a user and is designed to receive either
+String (the first application only) or {PlainText::ParseRule} (Array)
+and to return a fully (or partially) parsed {PlainText::ParseRule}.
+In short, the rule descries how to determine from where to where a
+paragraphs and boundaries are located, and maybe what and where the
+sections and sub-sections and so on are.
+
+For example, if a rule is Regexp, it describes how to split a String;
+it is applied to String in the first application, but if it is
+applied (and maybe registered as such) at the second or later stage,
+it is applied to each Paragraph and Section separately to split them further.
+
+{PlainText::ParseRule#apply} and {PlainText::Part.parse} are the
+standard methods to apply the rules to an object (either String or
+{PlainText::Part}.
+
+== Command-line tool
+
+=== countchar 
+
+Counts the number of characters in a file(s) or STDIN.
+
+The simplest example to run the command-line script is
+   countchar YourFile.txt
+
+You may start with
+   countchar --help
+to see the available options.
+
+== Miscellaneous
+
+Module {PlainText::Split} contains an instance method (and class
+method with the same name) {PlainText::Split#split_with_delimiter},
+which is included in String in default.  The method realises a
+reversible split of String with a delimiter of an arbitrary Regexp.
+
+In the standard String#split, the following is the result, when
+sent by a String instance +s+ = +"XQabXXcXQ"+:
+
+    s.split(/X+Q?/)         #=> ["", "ab", "c"],                   
+    s.split(/X+Q?/, -1)     #=> ["", "ab", "c", ""],               
+    s.split(/X+(Q?)/, -1)   #=> ["", "Q", "ab", "", "c", "Q", ""], 
+    s.split(/(X+(Q?))/, -1) #=> ["", "XQ", "Q", "ab", "XX", "", "c", "XQ", "Q", ""], 
+ 
+With this method,
+
+    s.split_with_delimiter(/X+(Q?)/)
+                            #=> ["", "XQ", "ab", "XX", "c", "XQ"]
+ 
+from which the original string is always easily recovered by simple +join+.
+
+Also, {PlainText::Util} contains some miscellaneous methods.
+
+== Description
+
+Work in progress...
+
+It is still in a preliminary state.
+
+== Install
+
+This script requires {Ruby}[http://www.ruby-lang.org] Version 2.0
+or above (possibley 2.2 or above?).
+
+As for the command-line script file, it can be put in any of your command-line search
+paths.  Make sure the RUBYLIB environment
+variable contains the library directory to this gem, which is
+   /THIS/GEM/LIBRARY/PATH/plain_text/lib
+
+You may need to modify the first line (Shebang line) of the script to suit your
+environment (it should be unnecessary for Linux and MacOS), or run it
+explicitly with your Ruby command as
+   Prompt% /YOUR/ENV/ruby /YOUR/INSTALLED/countchar
+
+== Developer's note
+
+=== Tests
+
+Ruby codes under the directory <tt>test/</tt> are the test scripts.
+You can run them from the top directory as <tt>ruby test/test_****.rb</tt>
+or simply run <tt>make test</tt>.
+
+
+== Known bugs
+
+None.
+
+
+== Copyright
+
+Author::  Masa Sakano < info a_t wisebabel dot com >
+Versions:: The versions of this package follow Semantic Versioning (2.0.0) http://semver.org/
+

data/Rakefile

@@ -0,0 +1,9 @@
+require 'rake/testtask'
+
+Rake::TestTask.new do |t|
+  t.libs << 'test'
+end
+
+desc "Run tests"
+task :default => :test
+

data/bin/countchar

@@ -0,0 +1,89 @@
+#!/usr/bin/env ruby
+# -*- coding: utf-8 -*-
+
+require 'optparse'
+require 'plain_text'
+
+BANNER = <<"__EOF__"
+USAGE: #{File.basename($0)} [options] [INFILE.txt] < STDIN
+  Print the number of characters in INFILE (or STDIN).
+__EOF__
+
+# Initialising the hash for the command-line options.
+OPTS = {
+  preserve_paragraph: true,
+  boundary_style: true,
+  lbs_style: :t, # :truncate,
+  lb_is_space: false,
+  sps_style: :truncate,
+  delete_asian_space: true,
+  linehead_style: :delete, 
+  linetail_style: :delete, 
+  firstsps_style: :delete,
+  lastsps_style:  :truncate,
+  line_i: nil,
+  line_f: nil,
+  # :chatter => 3,        # Default
+  debug: false,
+}
+
+# Function to handle the command-line arguments.
+#
+# ARGV will be modified, and the constant variable OPTS is set.
+#
+# @return [Hash]  Optional-argument hash.
+#
+def handle_argv
+  opt = OptionParser.new(BANNER)
+  opt.on(  '--[no-]preserve_paragraph', sprintf("Preserved paragraph structures? (Def: %s)", OPTS[:preserve_paragraph].inspect), TrueClass) {|v| OPTS[:preserve_paragraph] = v}
+  opt.on(  '--lbs-style=STYLE', sprintf("One of (t(runcate)|d(elete)|n(one)) (Def: truncate).", Symbol)) { |v| OPTS[:lbs_style]=v.strip[0].to_sym }
+  # opt.on(  '--version', "Display the version and exits.", TrueClass) {|v| OPTS[:version] = v}  # Consider opts.on_tail
+  opt.on(  '--[no-]debug', "Debug (Def: false)", TrueClass) {|v| OPTS[:debug] = v}
+  opt.separator ""        # Way to control a help message.
+  opt.separator "Note:"
+  opt.separator " Spaces are truncated in default."
+
+  opt.parse!(ARGV)
+
+  unless %i(t d n).include? OPTS[:lbs_style]
+    warn "ERROR: --lbs-style must be one of (t(runcate)|d(elete)|n(one))."; exit 1
+  end
+
+  OPTS
+end
+
+
+################################################
+# MAIN
+################################################
+
+$stdout.sync=true
+$stderr.sync=true
+
+class String
+  include PlainText
+end
+
+# Handle the command-line options => OPTS
+opts = handle_argv()
+
+str = ARGF.read
+
+puts str.count_char(
+       preserve_paragraph: opts[:preserve_paragraph],
+       boundary_style:     opts[:boundary_style],
+       lbs_style:          opts[:lbs_style],
+       lb_is_space:        opts[:lb_is_space],
+       sps_style:          opts[:sps_style],
+       delete_asian_space: opts[:delete_asian_space],
+       linehead_style:     opts[:linehead_style],
+       linetail_style:     opts[:linetail_style],
+       firstsps_style:     opts[:firstsps_style],
+       lastsps_style:      opts[:lastsps_style],
+     )
+
+exit
+
+__END__
+
+