pragmatic_segmenter_server 0.0.8 → 0.0.10

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 158a44e00c5a1ab64d3fb77001d48286637ed30cacb0d7098fb0077f311223ed
-  data.tar.gz: e78e2cc010710a465454c0bc339dd26edcde0f2da6637eccbc270c40fde3f78f
+  metadata.gz: 691daea57ce5a11eea96376091386612365d906b6fea70efa0db9b1233a0881f
+  data.tar.gz: a8bd66465a670f0b103740e113489fb047879bf26d6b1275c51b2f6efec1f724
 SHA512:
-  metadata.gz: 680f9f950efc494262237e9f0a604c01fec37b60272f48aa70cfe8f413475f4d9c4dcc435515431fddb4d756b40069ada85b1910f82c6a4e7b1d700fd132a069
-  data.tar.gz: 1b2221423b6ece5c40e066f61180502049a645aa3b5148a9fad325cc723e031234635cddd92399bed5da1708e40934596e0ef51ba39a3058cb536b358566a83e
+  metadata.gz: f374217088e3f83498d3cff70f655c99947fc66f06ff76f8665d7e2f93e9428e5e78f7556d978f1f0aef99b17ff8dc55172af889c2d95c9f75022886a787634f
+  data.tar.gz: c741cce30fc4e4041f722545834a27b29a69d331e6c0c15f66ae328c49bb461a818a9eec82a90bbf27c0822254da72e82edf8d7e2ec4fb48e741ecb0d2e93753

data/lib/pragmatic_segmenter_server/segment.rb

@@ -2,8 +2,10 @@ require 'pragmatic_segmenter'
 
 
 def segment(text, lang, segmentByNewline)
- 
-    newLineRegex = /((?: *[\n\r\t]+ *)+)/
+    
+    # Create parts
+    newLineRegex = /(?:( *[\n\r\t]+ *)+|( {3,}))/
+
     mask = ''
     segments = []
     
@@ -14,20 +16,40 @@ def segment(text, lang, segmentByNewline)
     end
   
     textParts.each do |textPart|
+     
         if segmentByNewline && textPart.match(newLineRegex)
             mask += textPart
+            # puts("Sep:" + textPart.inspect)
         else
+            # puts("texto:" + textPart )
             if textPart == ""
                 next
             end 
+
             ps = PragmaticSegmenter::Segmenter.new(text: textPart, language: lang, clean:false)
+
+     
             ps.segment.each do |segment|
                 segments.push(segment)
-            end				
-            mask += textPart.gsub(Regexp.new(ps.segment.map { |string| Regexp.escape(string) }.join("|")), "{}")
-
+      
+            end	
+            
+            # Created mask
+            isCleaned=false # Sometimes pragmatic "cleans" the text  
+            ps.segment.each do |segment|
+                if not textPart.include? segment
+                    isCleaned=true
+                    break
+                end
+            end	
+            if isCleaned
+                placeHolders = []
+                ps.segment.length.times { placeHolders << "{}" }
+                mask += placeHolders.join(" ")
+            else
+                mask += textPart.gsub(Regexp.new(ps.segment.map { |string| Regexp.escape(string) }.join("|")), "{}")
+            end
         end
-
     end
     
     return segments, mask

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: pragmatic_segmenter_server
 version: !ruby/object:Gem::Version
-  version: 0.0.8
+  version: 0.0.10
 platform: ruby
 authors:
 - Laurent Bié
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2023-04-27 00:00:00.000000000 Z
+date: 2023-05-02 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: sinatra
@@ -93,7 +93,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.1.2
+rubygems_version: 3.3.5
 signing_key: 
 specification_version: 4
 summary: A server for pragmatic segmenter