crawlab_ruby_sdk 0.1.0 → 0.1.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 53cc3b34e599f5a093bb5c53b8b2b8604667212b4cdde7b735a5d5eb281a9449
-  data.tar.gz: 1f99db5c5ad65dd23cb7de3c6d8132a651fe66e2e15f05091cbda36aa93c90dc
+  metadata.gz: ce64075649a810d878b0bcc3f9dd0e9c7de41d4fd9a2dece09d7cb0ce4afe075
+  data.tar.gz: 14f0c8c8005e724fff5d2cda3e4ae3b44f69e347a853c550cd422fdf285c1a33
 SHA512:
-  metadata.gz: 3661d366e9178bcf22c34424cc4dc481837d8b34c10a1d66f5d71a88d9119a4954bdb92b8b1f2f2d801dca581a6edb8f103ee9a00c965b6e69040c714ba1bd4b
-  data.tar.gz: 18959fafddab87d747f2a5af38a6ac52b94546132e8f5d150811d6170292786c4331da44fdd88f47e7a0a858a5e93c2316db1e417df67adcfc297c8052d2762d
+  metadata.gz: 591569d4dc1df08a0fd4d3ec4e6872c75d7b2d3dfb282e2c55fcd0db95c4ab7b85997292572ba4cb831b0ce8c706c1d593277224d7534dcfd4983923c2c4441f
+  data.tar.gz: 6b327871a439b12552792a215991acf6e15c36f4e05554bbe1412856018cbe93ec1465b393cde1347e43c977b7f96ccfa888a721552f859bb424c29fe493d290

data/crawlab_ruby_sdk.gemspec

@@ -6,11 +6,16 @@ Gem::Specification.new do |spec|
   spec.authors       = ["min"]
   spec.email         = ["lijinmin3903@126.com"]
 
-  spec.summary       = %q{This guy is very lazy.}
-  spec.description   = %q{This guy is very lazy.}
+  spec.summary       = %q{Write a short summary, because RubyGems requires one.}
+  spec.description   = %q{Write a longer description or delete this line.}
   spec.homepage      = "https://github.com/rich-bro/crawlab_ruby_sdk"
   spec.required_ruby_version = Gem::Requirement.new(">= 2.3.0")
 
+  spec.add_dependency 'grpc','~> 1.55.0'
+  spec.add_dependency 'google-protobuf','~> 3.23.2'
+  spec.add_dependency 'json','~> 2.6.3'
+  spec.add_dependency 'aliyun-sdk','~> 0.8.0'
+
   spec.metadata["allowed_push_host"] = "https://rubygems.org"
 
   spec.metadata["homepage_uri"] = spec.homepage

data/lib/client/oss_server_client.rb

@@ -0,0 +1,53 @@
+require 'aliyun/oss'
+
+class OssServerClient
+	attr_accessor :oss_client
+	attr_accessor :bucket
+
+	def initialize
+		get_oss_client
+		get_bucket
+	end
+
+	def get_oss_client
+		endpoint = ENV["CRAWLAB_OSS_ENDPOINT"]
+		access_key_id = ENV["CRAWLAB_OSS_ACCESS_KEY"]
+		access_key_secret = ENV["CRAWLAB_OSS_SECRET"]
+		bucket = ENV["CRAWLAB_OSS_BUCKET"]
+		if endpoint == nil || endpoint == ""
+			return
+		end
+		if access_key_id == nil || access_key_id == ""
+			return
+		end
+		if access_key_secret == nil || access_key_secret == ""
+			return
+		end
+
+		if bucket == nil || bucket == ""
+			return
+		end
+		@oss_client = Aliyun::OSS::Client.new(
+		  :endpoint => endpoint,
+		  :access_key_id => access_key_id,
+		:access_key_secret => access_key_secret)		
+	end
+
+	def get_bucket
+		bucket = ENV["CRAWLAB_OSS_BUCKET"]
+		if bucket == nil || bucket == ""
+			return
+		end
+		@bucket = @oss_client.get_bucket(bucket)
+	end
+
+	def send(oss_path,file_path)
+		if bucket == nil || @oss_client == nil
+			return "bucket is nil"
+		end
+		bucket.put_object(oss_path, :file => file_path)
+
+		bucket_url = bucket.object_url(oss_path)
+		return bucket_url
+	end
+end

data/lib/crawlab_ruby_sdk/version.rb

@@ -1,3 +1,3 @@
 module CrawlabRubySdk
-  VERSION = "0.1.0"
+  VERSION = "0.1.1"
 end

data/lib/crawlab_ruby_sdk.rb

@@ -9,21 +9,21 @@ def traverse_dir(file_path)
   end
 end
 
-PWD = File.expand_path("..", __FILE__)
+dir = File.expand_path("..", __FILE__)
 
-# puts PWD
+# puts dir
 
-traverse_dir(PWD+'/entity/stream_message_code_pb.rb')
-traverse_dir(PWD+'/entity/result_pb.rb')
-traverse_dir(PWD+'/entity/stream_message_pb.rb')
-traverse_dir(PWD+'/entity/stream_message_data_task_pb.rb')
-traverse_dir(PWD+'/client')
+traverse_dir(dir+'/entity/stream_message_code_pb.rb')
+traverse_dir(dir+'/entity/result_pb.rb')
+traverse_dir(dir+'/entity/stream_message_pb.rb')
+traverse_dir(dir+'/entity/stream_message_data_task_pb.rb')
+traverse_dir(dir+'/client')
 
 module CrawlabRubySdk
   class Error < StandardError; end
   # Your code goes here...
 
-
+  
   def self.save_item(item={})
     address = ENV["CRAWLAB_GRPC_ADDRESS"]
     if address==nil || address == ""
@@ -39,22 +39,72 @@ module CrawlabRubySdk
 
     sub_client = client.subscribe
 
-    task_id = ENV["CRAWLAB_TASK_ID"]
+    save(sub_client,[item])
+  end
+
+  def self.save_items(items=[])
+    address = ENV["CRAWLAB_GRPC_ADDRESS"]
+    if address==nil || address == ""
+      address = "localhost:9666"
+    end  
+
+    auth = ENV["CRAWLAB_GRPC_AUTH_KEY"]
+    if auth==nil || auth == ""
+      auth = "Crawlab2021!"
+    end  
+    
+    client = TaskServiceClient.new(address,auth)
+
+    sub_client = client.subscribe
+
+    save(sub_client,items)    
+  end
+
+  def self.save(sub_client,items = [])
+    results = []
+
+    items.each do |item|
+      results << item
+
+      if results.size >= 50
+        _save(sub_client,results)
+      end
+      
+    end
+
+    if results.size > 0
+      _save(sub_client,results)
+    end
+  end
+
+  def self._save(sub_client,items=[])
+    records = []
+    task_id = get_task_id
     if task_id == nil || task_id == ""
-      task_id = "6486e256fa1cb07a47c09adf"
+      return
     end
-    item = {"hello": "world","_tid":task_id}
-    item = Grpc::Result.new({"_tid":task_id,"name":"lala","age":"12"})
 
-    data = Grpc::StreamMessageDataTask.new()
-    # puts item
+    items.each do |item|
+      item["_tid"] = task_id
+      records << item
+    end
 
-    data.task_id = task_id
-    data.data.push(item)
+    data = {task_id: task_id,data:records}.to_json
+
+    msg = Grpc::StreamMessage.new(code:3,data:data)
+
+    sub_client.Send([msg]) 
+  end
+
+  def self.get_task_id
+    task_id = ENV["CRAWLAB_TASK_ID"]
+    if task_id == nil || task_id == ""
+      task_id = "6486e256fa1cb07a47c09adf"
+    end
+    return task_id
+  end
 
-    puts data
-    msg = Grpc::StreamMessage.new(code:3,data:data.to_json)
-    puts msg
-    sub_client.Send([msg])    
+  def self.save_file_to_oss(oss_path,file_path)
+    OssServerClient.new.send(oss_path,file_path)
   end
 end

data/test_grpc_client.rb

@@ -0,0 +1,17 @@
+
+require 'crawlab_ruby_sdk'
+def main
+	CrawlabRubySdk.save_item({name: "haha",age:12})
+
+	CrawlabRubySdk.save_items([{name: "haha",age:12},{name:"456",age:34}])
+	ENV.each do |k,v|
+		puts "#{k}=#{v}"
+	end
+
+	oss_path = "thinkthank_files/files/1123123123.pdf"
+	file_path = "/home/min/Downloads/RAND_RRA1218-2.pdf"
+	bucket_url = CrawlabRubySdk.save_file_to_oss(oss_path,file_path)
+	puts bucket_url
+end
+
+main

metadata

@@ -1,16 +1,72 @@
 --- !ruby/object:Gem::Specification
 name: crawlab_ruby_sdk
 version: !ruby/object:Gem::Version
-  version: 0.1.0
+  version: 0.1.1
 platform: ruby
 authors:
 - min
-autorequire:
+autorequire: 
 bindir: exe
 cert_chain: []
-date: 2023-06-14 00:00:00.000000000 Z
-dependencies: []
-description: This guy is very lazy.
+date: 2023-06-15 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: grpc
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 1.55.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 1.55.0
+- !ruby/object:Gem::Dependency
+  name: google-protobuf
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 3.23.2
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 3.23.2
+- !ruby/object:Gem::Dependency
+  name: json
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 2.6.3
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 2.6.3
+- !ruby/object:Gem::Dependency
+  name: aliyun-sdk
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.8.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.8.0
+description: Write a longer description or delete this line.
 email:
 - lijinmin3903@126.com
 executables: []
@@ -24,6 +80,7 @@ files:
 - bin/console
 - bin/setup
 - crawlab_ruby_sdk.gemspec
+- lib/client/oss_server_client.rb
 - lib/client/task_service_client.rb
 - lib/client/task_service_subscribe_client.rb
 - lib/crawlab_ruby_sdk.rb
@@ -39,6 +96,7 @@ files:
 - lib/entity/stream_message_pb.rb
 - lib/models/node_pb.rb
 - lib/models/task_pb.rb
+- test_grpc_client.rb
 homepage: https://github.com/rich-bro/crawlab_ruby_sdk
 licenses: []
 metadata:
@@ -46,7 +104,7 @@ metadata:
   homepage_uri: https://github.com/rich-bro/crawlab_ruby_sdk
   source_code_uri: https://github.com/rich-bro/crawlab_ruby_sdk
   changelog_uri: https://github.com/rich-bro/crawlab_ruby_sdk
-post_install_message:
+post_install_message: 
 rdoc_options: []
 require_paths:
 - lib
@@ -62,7 +120,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubygems_version: 3.1.4
-signing_key:
+signing_key: 
 specification_version: 4
-summary: This guy is very lazy.
+summary: Write a short summary, because RubyGems requires one.
 test_files: []