pragmatic_segmenter_server 0.0.8 → 0.0.9

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 158a44e00c5a1ab64d3fb77001d48286637ed30cacb0d7098fb0077f311223ed
-  data.tar.gz: e78e2cc010710a465454c0bc339dd26edcde0f2da6637eccbc270c40fde3f78f
+  metadata.gz: 9d32be051c0f866fa93fb7c5b25aab2949f704ea009fb374d8aa4356c7dbd2a8
+  data.tar.gz: b459035e2986c2383962836a705a36e75e933f879f90f45ca9055dce19e6a8af
 SHA512:
-  metadata.gz: 680f9f950efc494262237e9f0a604c01fec37b60272f48aa70cfe8f413475f4d9c4dcc435515431fddb4d756b40069ada85b1910f82c6a4e7b1d700fd132a069
-  data.tar.gz: 1b2221423b6ece5c40e066f61180502049a645aa3b5148a9fad325cc723e031234635cddd92399bed5da1708e40934596e0ef51ba39a3058cb536b358566a83e
+  metadata.gz: 46b4d760bea4ddcf94762aadb0eebcb0c38cefbab26010f74006e812c028715ce54a3193790e2367f2e79ef121fa65a1e5f3f7fa0f5e7f010e6ac9797d2e32aa
+  data.tar.gz: 7cf14771e1c810470c9fb82024353cf9e386a4c093ee3473689a812b039ccfd4af458b33f773c29e5a69a9b5046207ab14e6a7437e0d9a8f7a250f4fdbf30aea

data/lib/pragmatic_segmenter_server/segment.rb

@@ -2,8 +2,10 @@ require 'pragmatic_segmenter'
 
 
 def segment(text, lang, segmentByNewline)
- 
-    newLineRegex = /((?: *[\n\r\t]+ *)+)/
+    
+    # Create parts
+    newLineRegex = /(?:( *[\n\r\t]+ *)+|( {3,}))/
+
     mask = ''
     segments = []
     
@@ -14,20 +16,40 @@ def segment(text, lang, segmentByNewline)
     end
   
     textParts.each do |textPart|
+     
         if segmentByNewline && textPart.match(newLineRegex)
             mask += textPart
+            puts("Sep:" + textPart.inspect)
         else
+            puts("texto:" + textPart )
             if textPart == ""
                 next
             end 
+
             ps = PragmaticSegmenter::Segmenter.new(text: textPart, language: lang, clean:false)
+
+     
             ps.segment.each do |segment|
                 segments.push(segment)
-            end				
-            mask += textPart.gsub(Regexp.new(ps.segment.map { |string| Regexp.escape(string) }.join("|")), "{}")
-
+      
+            end	
+            
+            # Created mask
+            isCleaned=false # Sometimes pragmatic "cleans" the text  
+            ps.segment.each do |segment|
+                if not textPart.include? segment
+                    isCleaned=true
+                    break
+                end
+            end	
+            if isCleaned
+                placeHolders = []
+                ps.segment.length.times { placeHolders << "{}" }
+                mask += placeHolders.join(" ")
+            else
+                mask += textPart.gsub(Regexp.new(ps.segment.map { |string| Regexp.escape(string) }.join("|")), "{}")
+            end
         end
-
     end
     
     return segments, mask

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: pragmatic_segmenter_server
 version: !ruby/object:Gem::Version
-  version: 0.0.8
+  version: 0.0.9
 platform: ruby
 authors:
 - Laurent Bié
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2023-04-27 00:00:00.000000000 Z
+date: 2023-05-02 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: sinatra
@@ -93,7 +93,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.1.2
+rubygems_version: 3.3.5
 signing_key: 
 specification_version: 4
 summary: A server for pragmatic segmenter