pragmatic_segmenter_server 0.0.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 54321133f0512e94bc8df32b823780820dd57569
+  data.tar.gz: 595f906c6115a8cded9521529a66b4d2cca7c280
+SHA512:
+  metadata.gz: f92cf7292c893d7cb3ddc053a176007e9fe72022b7b6557e841e853032ccec9a5c7a8a495d2c714762c96617838d6a51e9099235af3c37fbaef651f8b7dc94d8
+  data.tar.gz: 4fb69b43f5f9af07e611038437e877706aece47b8c155c8dc4dd20e7463895b41812c574732199372d7bc19bcf19f9adaf2eaf0b7876ab3146118db12552cc07

data/lib/pragmatic_segmenter_server/segment.rb

@@ -0,0 +1,28 @@
+require 'pragmatic_segmenter'
+
+
+def segment(text, lang, segmentByNewline)
+    newLineRegex = /((?: *[\n\r\t]+ *)+)/
+    mask = ''
+    segments = []
+    
+    if segmentByNewline
+        textParts = text.split(newLineRegex)
+    else
+        textParts = [text]
+    end
+    
+    textParts.each do |textPart|
+        if segmentByNewline && textPart.match(newLineRegex)
+            mask += textPart
+        else
+            ps = PragmaticSegmenter::Segmenter.new(text: textPart, language: lang, clean:false)
+            ps.segment.each do |segment|
+                segments.push(segment)
+            end				
+            mask += textPart.gsub(Regexp.new(ps.segment.map { |string| Regexp.escape(string) }.join("|")), "{}")
+        end
+    end
+    
+    return segments, mask
+end

data/lib/pragmatic_segmenter_server.rb

@@ -0,0 +1,65 @@
+require 'sinatra'
+require 'json'
+require 'json-schema'
+require_relative 'segment'
+
+
+post '/segment' do
+	content_type :json
+
+	# Input 
+	input = JSON.parse(request.body.read)
+	
+	schema = {
+			"type" => "object",
+			"required" => ["texts", "lang"],
+			"properties" => {
+				"texts"  => {
+					"type" => "array",
+					"items": {
+						"type": "string",
+						"minItems": 1
+					}
+				},
+				"lang"  => {
+					"type" => "string"
+				},
+				"segmentByNewline" => {
+					"type" => "boolean"
+				}
+			}  
+		}
+
+	begin	
+		JSON::Validator.validate!(schema, input)
+	rescue JSON::Schema::ValidationError => e
+		json_message = {"error":e.message}.to_json
+		status 400
+		body json_message
+		return
+	end
+
+	# Extract info
+	lang = input["lang"]
+	texts =  input["texts"]
+
+	if !input.key?("segmentByNewline")
+		segmentByNewline = true
+	else
+		segmentByNewline = input["segmentByNewline"]
+	end
+
+		
+	# Segmentation
+	results = []
+	texts.each do |text, index|
+		segments, mask = segment(text, lang, segmentByNewline)
+		results.push({
+			"text": text,
+			"segments": segments,
+			"mask": mask
+		})
+	end
+
+	puts results.to_json
+end

metadata

@@ -0,0 +1,101 @@
+--- !ruby/object:Gem::Specification
+name: pragmatic_segmenter_server
+version: !ruby/object:Gem::Version
+  version: 0.0.1
+platform: ruby
+authors:
+- Laurent Bié
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2020-01-07 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: sinatra
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 2.0.3
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 2.0.3
+- !ruby/object:Gem::Dependency
+  name: sinatra-contrib
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 2.0.3
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 2.0.3
+- !ruby/object:Gem::Dependency
+  name: json-schema
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 2.8.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 2.8.0
+- !ruby/object:Gem::Dependency
+  name: pragmatic_segmenter
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 0.3.18
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 0.3.18
+description: A HTTP server for pragmatic segmenter
+email: l.bie@pangeanic.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- lib/pragmatic_segmenter_server.rb
+- lib/pragmatic_segmenter_server/segment.rb
+homepage: ''
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.5.2.1
+signing_key: 
+specification_version: 4
+summary: A server for pragmatic segmenter
+test_files: []