pragmatic_segmenter_server 0.0.7 → 0.0.9

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 004a705fc7f50fa65c8b5a99af899659baad350ffbe8992d6b929fccf2fc6195
-  data.tar.gz: a713bffdbed2ae5a80fd7481325d75d9e9e78fe47259268fedd5f8c18ff1e8e5
+  metadata.gz: 9d32be051c0f866fa93fb7c5b25aab2949f704ea009fb374d8aa4356c7dbd2a8
+  data.tar.gz: b459035e2986c2383962836a705a36e75e933f879f90f45ca9055dce19e6a8af
 SHA512:
-  metadata.gz: 24286ee9393591d3c216a12eff44bd68a498db13af431cf187d208efe1491997512b558a265ec2e1a64b3f4eb07b755e90141691d3bf6d89865cae8cfeeea7f9
-  data.tar.gz: e82df4506646404fcff2d5d637ed4062f94393b8d2b1e68c2b99be304c1a7ac46cc9ac416bb5669bc02adbb443aeb2caae8ad271cf7eb64d67f737063371cf6f
+  metadata.gz: 46b4d760bea4ddcf94762aadb0eebcb0c38cefbab26010f74006e812c028715ce54a3193790e2367f2e79ef121fa65a1e5f3f7fa0f5e7f010e6ac9797d2e32aa
+  data.tar.gz: 7cf14771e1c810470c9fb82024353cf9e386a4c093ee3473689a812b039ccfd4af458b33f773c29e5a69a9b5046207ab14e6a7437e0d9a8f7a250f4fdbf30aea

data/lib/pragmatic_segmenter_server/segment.rb

@@ -2,8 +2,10 @@ require 'pragmatic_segmenter'
 
 
 def segment(text, lang, segmentByNewline)
- 
-    newLineRegex = /((?: *[\n\r\t]+ *)+)/
+    
+    # Create parts
+    newLineRegex = /(?:( *[\n\r\t]+ *)+|( {3,}))/
+
     mask = ''
     segments = []
     
@@ -14,20 +16,40 @@ def segment(text, lang, segmentByNewline)
     end
   
     textParts.each do |textPart|
+     
         if segmentByNewline && textPart.match(newLineRegex)
             mask += textPart
+            puts("Sep:" + textPart.inspect)
         else
+            puts("texto:" + textPart )
             if textPart == ""
                 next
             end 
+
             ps = PragmaticSegmenter::Segmenter.new(text: textPart, language: lang, clean:false)
+
+     
             ps.segment.each do |segment|
                 segments.push(segment)
-            end				
-            mask += textPart.gsub(Regexp.new(ps.segment.map { |string| Regexp.escape(string) }.join("|")), "{}")
-
+      
+            end	
+            
+            # Created mask
+            isCleaned=false # Sometimes pragmatic "cleans" the text  
+            ps.segment.each do |segment|
+                if not textPart.include? segment
+                    isCleaned=true
+                    break
+                end
+            end	
+            if isCleaned
+                placeHolders = []
+                ps.segment.length.times { placeHolders << "{}" }
+                mask += placeHolders.join(" ")
+            else
+                mask += textPart.gsub(Regexp.new(ps.segment.map { |string| Regexp.escape(string) }.join("|")), "{}")
+            end
         end
-
     end
     
     return segments, mask

data/lib/pragmatic_segmenter_server.rb

@@ -65,4 +65,5 @@ post '/segment' do
 end
 
 get '/healthcheck' do
-	'ok'
+	'ok'
+end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: pragmatic_segmenter_server
 version: !ruby/object:Gem::Version
-  version: 0.0.7
+  version: 0.0.9
 platform: ruby
 authors:
 - Laurent Bié
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2023-04-27 00:00:00.000000000 Z
+date: 2023-05-02 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: sinatra
@@ -93,7 +93,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.1.2
+rubygems_version: 3.3.5
 signing_key: 
 specification_version: 4
 summary: A server for pragmatic segmenter