glaemscribe 1.1.14 → 1.3.0

data/lib/api/mode_parser.rb

@@ -83,7 +83,11 @@ module Glaemscribe
         doc.root_node.gpath("preprocessor.if").each{ |e| validate_presence_of_args(e,  1) }    
         doc.root_node.gpath("preprocessor.elsif").each{ |e| validate_presence_of_args(e,  1) }    
         doc.root_node.gpath("postprocessor.if").each{ |e| validate_presence_of_args(e,  1) }    
-        doc.root_node.gpath("postprocessor.elsif").each{ |e| validate_presence_of_args(e,  1) }    
+        doc.root_node.gpath("postprocessor.elsif").each{ |e| validate_presence_of_args(e,  1) }  
+        
+        doc.root_node.children.each { |c|
+          @mode.errors << Glaeml::Error.new(c.line, "'if' conditions are not allowed in that scope.") if c.name == 'if'
+        }  
       end
       
       def create_if_cond_for_if_term(line, if_term, cond)      
@@ -94,7 +98,13 @@ module Glaemscribe
         ifcond            
       end
       
-      def traverse_if_tree(root_code_block, root_element, text_procedure, element_procedure)
+      def traverse_if_tree(context, text_procedure, element_procedure)
+
+        owner         = context[:owner]           # The root object of the if tree
+        root_element  = context[:root_element]    # The glaeml root_element of that if tree
+        rule_group    = context[:rule_group]      # The rule group in which this traversal happens (may be null for pre/post processors)
+
+        root_code_block           = owner.root_code_block
         current_parent_code_block = root_code_block
         
         root_element.children.each{ |child|
@@ -145,7 +155,65 @@ module Glaemscribe
               end
               
               current_parent_code_block       = if_term.parent_code_block
+            when 'macro'
+        
+              # Macro definition, cannot be defined in conditional blocks
+              if current_parent_code_block.parent_if_cond || root_element.name != "rules"
+                @mode.errors << Glaeml::Error.new(child.line, "Macros can only defined in the 'rules' scope, not in a conditional block (because they are replaced and used at parsing time) or a macro block (local macros are not handled).")
+                return
+              end
+              
+              if !child.args || child.args.count == 0
+                @mode.errors << Glaeml::Error.new(child.line, "Macro misses a name.")
+                return   
+              end
+
+              macro_args = child.args.clone
+              macro_name = macro_args.shift
+              macro_args.each{ |arg|
+                if(!arg =~ /[0-9A-Z_]+/)
+                  @mode.errors << Glaeml::Error.new(child.line, "Macro argument name #{arg} has wrong format.")
+                  return
+                end
+              }
+              
+              if rule_group.macros[macro_name]
+                @mode.errors << Glaeml::Error.new(child.line, "Redefining macro #{macro_name}.")
+                return
+              end
               
+              macro = Macro.new(rule_group,macro_name,macro_args)
+              macro_context = {:owner => macro, :root_element => child, :rule_group => rule_group}
+              traverse_if_tree(macro_context, text_procedure, element_procedure)
+
+              rule_group.macros[macro_name] = macro
+
+            when 'deploy'
+
+              if !rule_group
+                @mode.errors << Glaeml::Error.new(child.line, "Macros can only be deployed in a rule group.")
+                return
+              end
+
+              macro_args = child.args.clone
+              macro_name = macro_args.shift
+              macro      = rule_group.macros[macro_name]
+
+              if !macro
+                @mode.errors << Glaeml::Error.new(child.line, "Macro '#{macro_name}' not found in rule group '#{rule_group.name}'.")
+                return
+              end
+
+              wanted_argcount = macro.arg_names.count
+              given_argcount  = macro_args.count
+              if wanted_argcount != given_argcount
+                @mode.errors << Glaeml::Error.new(child.line, "Macro '#{macro_name}' takes #{wanted_argcount} arguments, not #{given_argcount}.")
+                return
+              end
+
+              macro_node = IfTree::MacroDeployTerm.new(macro, child.line, current_parent_code_block, macro_args)
+              current_parent_code_block.terms << macro_node
+
             else
               # Do something with this child element
               element_procedure.call(current_parent_code_block, child)            
@@ -181,13 +249,16 @@ module Glaemscribe
           if !operator_class
             @mode.errors << Glaeml::Error.new(element.line,"Operator #{operator_name} is unknown.")
           else            
-            term.operators << operator_class.new(element.clone)
+            term.operators << operator_class.new(@mode, element.clone)
           end  
         }  
-        
-        root_code_block = ((pre_not_post)?(@mode.pre_processor.root_code_block):(@mode.post_processor.root_code_block))
-               
-        self.traverse_if_tree(root_code_block, processor_element, text_procedure, element_procedure )                       
+
+        processor_context = {
+            owner: ((pre_not_post)?(@mode.pre_processor):(@mode.post_processor)),
+            root_element: processor_element,
+            rule_group: nil
+        }
+        traverse_if_tree(processor_context, text_procedure, element_procedure )
       end
       
       def parse(file_path, mode_options = {})
@@ -219,6 +290,7 @@ module Glaemscribe
         doc.root_node.gpath("options.option").each{ |option_element|
           values      = {}
           visibility  = nil
+          is_radio    = false
   
           option_element.gpath("value").each{ |value_element|
             value_name                = value_element.args.first
@@ -227,6 +299,8 @@ module Glaemscribe
           option_element.gpath("visible_when").each{ |visible_element|
             visibility = visible_element.args.first
           }
+          
+          option_element.gpath('radio').each{|e| is_radio = true}
         
           option_name_at          = option_element.args[0]
           option_default_val_at   = option_element.args[1]
@@ -236,8 +310,9 @@ module Glaemscribe
             @mode.errors << Glaeml::Error.new(option_element.line, "Missing option default value.")
           end
             
-          option        = Option.new(@mode, option_name_at, option_default_val_at, values, visibility)
-          @mode.options[option.name] = option
+          option                      = Option.new(@mode, option_name_at, option_default_val_at, values, option_element.line, visibility)
+          option.is_radio             = is_radio
+          @mode.options[option.name]  = option
         }
         
         # Read the supported font list
@@ -301,6 +376,7 @@ module Glaemscribe
                                    
             lcount  = element.line
             element.args[0].lines.to_a.each{ |l| 
+              # Split into lines of code and count the lines
               l       = l.strip
               term.code_lines << IfTree::CodeLine.new(l, lcount)           
               lcount  += 1                         
@@ -310,13 +386,40 @@ module Glaemscribe
           element_procedure = Proc.new { |current_parent_code_block, element|
             # This is fatal.
             @mode.errors << Glaeml::Error.new(element.line, "Unknown directive #{element.name}.")      
-          }  
-          
-          self.traverse_if_tree( rule_group.root_code_block, rules_element, text_procedure, element_procedure )                 
+          }
+
+          processor_context = {
+              owner:           rule_group,
+              root_element:   rules_element,
+              rule_group:     rule_group
+          }
+          traverse_if_tree(processor_context, text_procedure, element_procedure )
         }
-                                                         
-        @mode.finalize(mode_options) if !@mode.errors.any?
+
+        espeak_option = @mode.options['espeak_voice']
+        if espeak_option
+          # Singleton lazy load the TTS engine
+          # If the mode relies on espeak
+          @mode.has_tts = true
+          
+          begin
+            TTS::load_engine
+            
+            # Check if all voices are supported
+            espeak_option.values.keys.each { |vname|
+              voice = TTS::option_name_to_voice(vname)
+              if !(TTS::voice_list.include? voice)
+                @mode.errors << Glaeml::Error.new(espeak_option.line, "Option has unhandled voice #{voice}.")   
+              end
+            }
+          rescue
+            @mode.errors << Glaeml::Error.new(espeak_option.line, "Failed to load TTS engine.")  
+          end
+
+        end
         
+        @mode.finalize(mode_options) if !@mode.errors.any?
+    
         @mode
       end
     end

data/lib/api/object_additions.rb

@@ -1,7 +1,9 @@
 # From http://blade.nagaokaut.ac.jp/cgi-bin/scat.rb/ruby/ruby-list/43424
 class Object
+=begin
   def deep_clone
     return @deep_cloning_obj if @deep_cloning
+      
     @deep_cloning_obj = clone
     @deep_cloning_obj.instance_variables.each do |var|
       val = @deep_cloning_obj.instance_variable_get(var)
@@ -19,4 +21,24 @@ class Object
     @deep_cloning_obj = nil
     deep_cloning_obj
   end
-end
+=end
+  
+  def deep_clone(cache={})
+    return cache[self] if cache.key?(self)
+
+    copy = clone()
+    cache[self] = copy
+
+    copy.instance_variables.each do |var|
+      val = instance_variable_get(var)
+      begin
+        val = val.deep_clone(cache)
+      rescue TypeError
+        next
+      end
+      copy.instance_variable_set(var, val)
+    end
+
+    return copy
+  end  
+end

data/lib/api/option.rb

@@ -25,24 +25,29 @@ module Glaemscribe
     
     class Option
       attr_reader   :mode
+      attr_reader   :line
       attr_reader   :name
       attr_reader   :type
       attr_reader   :default_value_name      
       attr_reader   :values
+      
+      attr_accessor :is_radio
 
-    
       class Type
         ENUM = "ENUM"
         BOOL = "BOOL"
       end
          
-      def initialize(mode, name, default_value_name, values, visibility = nil)
+      def initialize(mode,  name, default_value_name, values, line, visibility = nil)
         @mode               = mode
         @name               = name
         @default_value_name = default_value_name
         @type               = (values.keys.count == 0)?(Type::BOOL):(Type::ENUM)
         @values             = values
         @visibility         = visibility
+        @line               = line
+        @value_to_names = {}
+        @values.each { |vname, val| @value_to_names[val] = vname }
       end
       
       def default_value
@@ -63,6 +68,16 @@ module Glaemscribe
         end
       end
       
+      def value_name_for_value(value)
+        if @type == Type::BOOL
+          return "true"  if value == true  || value == "true"
+          return "false" if value == false || value == "false"
+          return nil
+        else
+          return @value_to_names[value]
+        end
+      end
+      
       def visible?
         if_eval = Eval::Parser.new()
         

data/lib/api/post_processor/outspace.rb

@@ -0,0 +1,44 @@
+# encoding: UTF-8
+#
+# Glǽmscribe (also written Glaemscribe) is a software dedicated to
+# the transcription of texts between writing systems, and more 
+# specifically dedicated to the transcription of J.R.R. Tolkien's 
+# invented languages to some of his devised writing systems.
+# 
+# Copyright (C) 2015 Benjamin Babut (Talagan).
+# 
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# any later version.
+# 
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+# 
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+# A post processor operator to replace the out_space on the fly.
+# This has the same effect as the \outspace parameter
+# But can be included in the postprocessor and benefit from the if/then logic
+
+module Glaemscribe
+  module API
+    
+    class OutspacePostProcessorOperator < PostProcessorOperator
+      def initialize(mode, glaeml_element)
+        super(mode, glaeml_element)
+        @out_space = @mode.post_processor.out_space  = glaeml_element.args[0].split.reject{|token| token.empty? }   
+      end
+      
+      def apply(tokens, charset)
+        @mode.post_processor.out_space = @out_space
+        tokens
+      end
+    end
+
+    ResourceManager::register_post_processor_class("outspace", OutspacePostProcessorOperator)    
+  end
+end

data/lib/api/post_processor/resolve_virtuals.rb

@@ -50,19 +50,35 @@ module Glaemscribe
           # Try to replace
           last_trigger = @last_triggers[c]
           if last_trigger != nil
-            new_tokens[idx] = last_trigger.names.first # Take the first name of the non-virtual replacement.
+            new_tokens[idx] = last_trigger.names.first  # Take the first name of the non-virtual replacement.
+            token           = new_tokens[idx]           # Consider the token replaced, being itself a potential trigger for further virtuals (cascading virtuals)
           end
-        else
-          # Update states of virtual classes
-          charset.virtual_chars.each{|vc|
-            rc                  = vc[token]
-            @last_triggers[vc]  = rc if rc != nil 
-          }
-        end        
+        end
+        
+        # Update states of virtual classes
+        charset.virtual_chars.each{|vc|
+          rc                  = vc[token]
+          @last_triggers[vc]  = rc if rc != nil 
+        }
+      end
+      
+      def apply_sequences(charset,tokens)
+        ret = []
+        tokens.each { |token|
+          c = charset[token]
+          if c && c.sequence?
+            ret += c.sequence
+          else
+            ret << token
+          end
+        }
+        ret
       end
       
       def apply(tokens,charset)
-        
+        # Apply sequence chars
+        tokens = apply_sequences(charset,tokens)
+      
         # Clone the tokens so that we can perform ligatures AND diacritics without interferences
         new_tokens = tokens.clone
         

data/lib/api/resource_manager.rb

@@ -23,6 +23,7 @@ module Glaemscribe
   module API
     module ResourceManager
         
+      # The same structure is kept within the gem
       MODE_PATH     = File.dirname(__FILE__) + "/../../glaemresources/modes/"
       MODE_EXT      = "glaem"
     

data/lib/api/rule_group.rb

@@ -22,35 +22,92 @@
 
 module Glaemscribe
   module API
+    
+    class RuleGroupVar
+      attr_reader :name, :value
+      
+      def initialize(name, value, is_pointer)
+        @name       = name
+        @value      = value
+        @is_pointer = is_pointer
+      end
+      
+      def pointer?
+        is_pointer
+      end
+    end
+    
     class RuleGroup
   
-      VAR_NAME_REGEXP     = /{([0-9A-Z_]+)}/
-      VAR_DECL_REGEXP     = /^\s*{([0-9A-Z_]+)}\s+===\s+(.+?)\s*$/
-      RULE_REGEXP         = /^\s*(.*?)\s+-->\s+(.+?)\s*$/
-      CROSS_RULE_REGEXP   = /^\s*(.*?)\s+-->\s+([\s0-9,]+)\s+-->\s+(.+?)\s*$/
+      VAR_NAME_REGEXP             = /{([0-9A-Z_]+)}/
+      UNICODE_VAR_NAME_REGEXP_IN  = /^UNI_([0-9A-F]+)$/
+      UNICODE_VAR_NAME_REGEXP_OUT = /{UNI_([0-9A-F]+)}/
+      
+      VAR_DECL_REGEXP             = /^\s*{([0-9A-Z_]+)}\s+===\s+(.+?)\s*$/
+      POINTER_VAR_DECL_REGEXP     = /^\s*{([0-9A-Z_]+)}\s+<=>\s+(.+?)\s*$/
+      RULE_REGEXP                 = /^\s*(.*?)\s+-->\s+(.+?)\s*$/
+      
+      CROSS_SCHEMA_REGEXP         = /[0-9]+(\s*,\s*[0-9]+)*/
+      
+      CROSS_RULE_REGEXP           = /^\s*(.*?)\s+-->\s+(#{CROSS_SCHEMA_REGEXP}|#{VAR_NAME_REGEXP}|identity)\s+-->\s+(.+?)\s*$/
   
-      attr_reader :root_code_block, :name, :mode, :in_charset, :rules
+      attr_reader :root_code_block, :name, :mode, :in_charset, :rules, :macros
   
       def initialize(mode,name)
         @name             = name
         @mode             = mode
+        @macros           = {}
         @root_code_block  = IfTree::CodeBlock.new       
       end
   
-      def add_var(var_name, value)
-        @vars[var_name] = value
+      def add_var(var_name, value, is_pointer)
+        @vars[var_name] = RuleGroupVar.new(var_name, value, is_pointer)
       end
-  
+      
       # Replace all vars in expression
-      def apply_vars(line, string)
-        ret = string.gsub(VAR_NAME_REGEXP) { |cap_var|
-          rep = @vars[$1]
-          if !rep
-            @mode.errors << Glaeml::Error.new(line, "In expression: #{string}: failed to evaluate variable: #{cap_var}.") 
+      def apply_vars(line, string, allow_unicode_vars=false)
+        
+        ret               = string
+        stack_depth       = 0
+        had_replacements  = true
+        
+        while had_replacements
+          
+          had_replacements = false
+          ret = ret.gsub(VAR_NAME_REGEXP) { |cap_var|
+            vname = $1
+            v     = @vars[vname]
+            if !v
+              if vname =~ UNICODE_VAR_NAME_REGEXP_IN
+                # A unicode variable.
+                if allow_unicode_vars
+                  # Just keep this variable intact, it will be replaced at the last moment of the parsing
+                  rep = cap_var
+                else
+                  @mode.errors << Glaeml::Error.new(line, "In expression: #{string}: making wrong use of unicode variable: #{cap_var}. Unicode vars can only be used in source members of a rule or in the definition of another variable.") 
+                  return nil
+                end
+              else
+                @mode.errors << Glaeml::Error.new(line, "In expression: #{string}: failed to evaluate variable: #{cap_var}.") 
+                return nil
+              end
+            else
+              rep = v.value
+              # Only count replacements on non unicode vars
+              had_replacements = true
+            end
+            rep
+          }
+          stack_depth += 1
+                           
+          break if !had_replacements
+          
+          if stack_depth > 16
+            @mode.errors << Glaeml::Error.new(line, "In expression: #{string}: evaluation stack overflow.") 
             return nil
           end
-          rep
-        }
+        end
+                
         ret
       end
       
@@ -60,6 +117,58 @@ module Glaemscribe
             term.code_lines.each{ |cl|
               finalize_code_line(cl) 
             } 
+          elsif(term.is_macro_deploy?)
+                      
+            # Ok this is a bit dirty but I don't want to rewrite the error managamenet
+            # So add an error and if it's still the last (meaning there were no error) one remove it
+            possible_error = Glaeml::Error.new(term.line, ">> Macro backtrace : #{term.macro.name}")
+            @mode.errors << possible_error
+            
+            # First, test if variable is pushable
+            arg_values = []
+            term.macro.arg_names.each_with_index { |arg_name, i|
+              
+              var_value = nil
+              
+              if @vars[arg_name]
+                @mode.errors << Glaeml::Error.new(term.line, "Local variable #{arg_name} hinders a variable with the same name in this context. Use only local variable names in macros!")
+              else
+                # Evaluate local var
+                var_value_ex  = term.arg_value_expressions[i]
+                var_value     = apply_vars(term.line, var_value_ex, true)      
+                
+                if !var_value
+                  @mode.errors << Glaeml::Error.new(term.line, "Thus, variable {#{arg_name}} could not be declared.")
+                end              
+              end
+            
+              arg_values << {name: arg_name, val: var_value}
+            }
+            
+            # We push local vars after the whole loop to avoid interferences between them when evaluating them
+            arg_values.each { |v| 
+              if v[:val]
+                add_var(v[:name],v[:val],false)
+              end
+            }
+        
+            descend_if_tree(term.macro.root_code_block, trans_options)
+            
+            # Remove the local vars from the scope (only if they were leggit)
+            arg_values.each { |v| 
+              if v[:val]
+                @vars[v[:name]] = nil
+              end
+            }
+                      
+            if mode.errors.last == possible_error
+              # Remove the error scope if there were no errors
+              mode.errors.pop
+            else
+              # Add another one to close the context
+              @mode.errors << Glaeml::Error.new(term.line, "<< Macro backtrace : #{term.macro.name}")
+            end
+                        
           else
             term.if_conds.each{ |if_cond|
               
@@ -81,8 +190,8 @@ module Glaemscribe
             
       def finalize_rule(line, match_exp, replacement_exp, cross_schema = nil)
         
-        match                 = apply_vars(line, match_exp)
-        replacement           = apply_vars(line, replacement_exp)
+        match                 = apply_vars(line, match_exp, true)
+        replacement           = apply_vars(line, replacement_exp, false)
         
         return if !match || !replacement # Failed
               
@@ -102,20 +211,42 @@ module Glaemscribe
 
             var_name      = $1
             var_value_ex  = $2
-            var_value     = apply_vars(code_line.line, var_value_ex)
+            var_value     = apply_vars(code_line.line, var_value_ex, true)
         
             if !var_value
               @mode.errors << Glaeml::Error.new(code_line.line, "Thus, variable {#{var_name}} could not be declared.")
               return
             end
         
-            add_var(var_name,var_value)
+            add_var(var_name, var_value, false)
  
+          elsif code_line.expression =~ POINTER_VAR_DECL_REGEXP
+            
+            var_name      = $1
+            var_value_ex  = $2
+       
+            add_var(var_name, var_value_ex, true)
+            
           elsif code_line.expression =~ CROSS_RULE_REGEXP
         
             match         = $1
             cross         = $2
-            replacement   = $3      
+            var_name      = $4
+            replacement   = $5      
+            
+            if var_name
+              # This was a variable declaration           
+              var_value = apply_vars(code_line.line, cross, false)
+              if !var_value
+                @mode.errors << Glaeml::Error.new(code_line.line, "Thus, variable {#{var_name}} could not be declared.")
+                return
+              end
+              cross = var_value
+            end
+            
+            if cross == "identity"
+              cross = nil              
+            end
       
             finalize_rule(code_line.line, match, replacement, cross)
       
@@ -139,11 +270,24 @@ module Glaemscribe
         @in_charset       = {}
         @rules            = []
         
-        add_var("NULL","")
+        add_var("NULL","",false)
   
-        add_var("UNDERSCORE", SPECIAL_CHAR_UNDERSCORE)
-        add_var("NBSP",       SPECIAL_CHAR_NBSP)
-        
+        # Characters that are not easily entered or visible in a text editor
+        add_var("NBSP",           "{UNI_A0}",   false)
+        add_var("WJ",             "{UNI_2060}", false)
+        add_var("ZWSP",           "{UNI_200B}", false)
+        add_var("ZWNJ",           "{UNI_200C}", false)        
+
+        # The following characters are used by the mode syntax.
+        # Redefine some convenient tools.
+        add_var("UNDERSCORE",     "{UNI_5F}",  false)
+        add_var("ASTERISK",       "{UNI_2A}",  false)
+        add_var("COMMA",          "{UNI_2C}",  false)
+        add_var("LPAREN",         "{UNI_28}",  false)
+        add_var("RPAREN",         "{UNI_29}",  false)
+        add_var("LBRACKET",       "{UNI_5B}",  false)
+        add_var("RBRACKET",       "{UNI_5D}",  false)
+       
         descend_if_tree(@root_code_block, trans_options)
                             
         # Now that we have selected our rules, create the in_charset of the rule_group 
@@ -151,8 +295,8 @@ module Glaemscribe
           r.sub_rules.each { |sr|
             sr.src_combination.join("").split(//).each{ |inchar|
               # Add the character to the map of input characters
-              # Ignore '_' (bounds of word) and '|' (word breaker)
-              @in_charset[inchar] = self if inchar != WORD_BREAKER && inchar != WORD_BOUNDARY
+              # Ignore '\u0000' (bounds of word) and '|' (word breaker)
+              @in_charset[inchar] = self if inchar != WORD_BREAKER && inchar != WORD_BOUNDARY_TREE
             }
           }
         }

data/lib/api/sheaf_chain_iterator.rb

@@ -108,7 +108,7 @@ module Glaemscribe
         return false
       end
   
-      # Calculate all cominations for the chain
+      # Calculate all cominations for the chain, for the current iterator value
       def combinations
         resolved = []
         @iterators.each_with_index{ |counter, index|

data/lib/api/transcription_pre_post_processor.rb

@@ -27,7 +27,8 @@ module Glaemscribe
       attr_reader :glaeml_element
       attr_reader :finalized_glaeml_element
       
-      def initialize(glaeml_element)
+      def initialize(mode, glaeml_element)
+        @mode           = mode
         @glaeml_element = glaeml_element
       end
       
@@ -121,15 +122,17 @@ module Glaemscribe
       attr_accessor :out_space
           
       def apply(tokens, out_charset)
-        
-        out_space_str     = " "
-        out_space_str     = @out_space.map{ |token| out_charset[token].str }.join("") if @out_space
          
         # Apply filters
         @operators.each{ |operator|
           tokens = operator.apply(tokens,out_charset)
         } 
-        
+
+        out_space_str     = " "
+        out_space_str     = @out_space.map{ |token| 
+          out_charset[token]&.str || UNKNOWN_CHAR_OUTPUT
+        }.join("") if @out_space
+
         # Convert output
         ret = ""
         tokens.each{ |token|