crawlab_ruby_sdk 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: 53cc3b34e599f5a093bb5c53b8b2b8604667212b4cdde7b735a5d5eb281a9449
+  data.tar.gz: 1f99db5c5ad65dd23cb7de3c6d8132a651fe66e2e15f05091cbda36aa93c90dc
+SHA512:
+  metadata.gz: 3661d366e9178bcf22c34424cc4dc481837d8b34c10a1d66f5d71a88d9119a4954bdb92b8b1f2f2d801dca581a6edb8f103ee9a00c965b6e69040c714ba1bd4b
+  data.tar.gz: 18959fafddab87d747f2a5af38a6ac52b94546132e8f5d150811d6170292786c4331da44fdd88f47e7a0a858a5e93c2316db1e417df67adcfc297c8052d2762d

data/.gitignore

@@ -0,0 +1,10 @@
+/.bundle/
+/.yardoc
+/_yardoc/
+/coverage/
+/doc/
+/pkg/
+/spec/reports/
+/tmp/
+*.gem
+Gemfile.lock

data/Gemfile

@@ -0,0 +1,6 @@
+source "https://rubygems.org"
+
+# Specify your gem's dependencies in crawlab_ruby_sdk.gemspec
+gemspec
+
+gem "rake", "~> 12.0"

data/README.md

@@ -0,0 +1,49 @@
+# CrawlabRubySdk
+
+Welcome to your new gem! In this directory, you'll find the files you need to be able to package up your Ruby library into a gem. Put your Ruby code in the file `lib/crawlab_ruby_sdk`. To experiment with that code, run `bin/console` for an interactive prompt.
+
+TODO: Delete this and the text above, and describe your gem
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+```ruby
+gem 'crawlab_ruby_sdk'
+```
+
+And then execute:
+
+    $ bundle install
+
+Or install it yourself as:
+
+    $ gem install crawlab_ruby_sdk
+
+## Usage
+
+CrawlabRubySdk.save_item(item={})
+
+## Development
+
+After checking out the repo, run `bin/setup` to install dependencies. You can also run `bin/console` for an interactive prompt that will allow you to experiment.
+
+To install this gem onto your local machine, run `bundle exec rake install`. To release a new version, update the version number in `version.rb`, and then run `bundle exec rake release`, which will create a git tag for the version, push git commits and tags, and push the `.gem` file to [rubygems.org](https://rubygems.org).
+
+## Contributing
+
+Bug reports and pull requests are welcome on GitHub at https://github.com/[USERNAME]/crawlab_ruby_sdk.
+
+**bundle gem crawlab_ruby_sdk.gemspec**
+**添加代码**
+**gem build**
+**gem install --local crawlab_ruby_sdk-0.1.0.gem**
+
+curl -u ***** https://rubygems.org/api/v1/api_key.yaml # ***** 为你的用户名
+
+$ rake -T
+rake build    # Build blomming_api-0.3.7.gem into the pkg directory
+rake install  # Build and install blomming_api-0.3.7.gem into system gems
+rake release  # Create tag v0.3.7 and build and push blomming_api-0.3.7.gem to Rubygems
+
+

data/Rakefile

@@ -0,0 +1,2 @@
+require "bundler/gem_tasks"
+task :default => :spec

data/bin/console

@@ -0,0 +1,14 @@
+#!/usr/bin/env ruby
+
+require "bundler/setup"
+require "crawlab_ruby_sdk"
+
+# You can add fixtures and/or initialization code here to make experimenting
+# with your gem easier. You can also use a different console, if you like.
+
+# (If you use this, don't forget to add pry to your Gemfile!)
+# require "pry"
+# Pry.start
+
+require "irb"
+IRB.start(__FILE__)

data/bin/setup

@@ -0,0 +1,8 @@
+#!/usr/bin/env bash
+set -euo pipefail
+IFS=$'\n\t'
+set -vx
+
+bundle install
+
+# Do any other automated setup that you need to do here

data/crawlab_ruby_sdk.gemspec

@@ -0,0 +1,28 @@
+require_relative 'lib/crawlab_ruby_sdk/version'
+
+Gem::Specification.new do |spec|
+  spec.name          = "crawlab_ruby_sdk"
+  spec.version       = CrawlabRubySdk::VERSION
+  spec.authors       = ["min"]
+  spec.email         = ["lijinmin3903@126.com"]
+
+  spec.summary       = %q{This guy is very lazy.}
+  spec.description   = %q{This guy is very lazy.}
+  spec.homepage      = "https://github.com/rich-bro/crawlab_ruby_sdk"
+  spec.required_ruby_version = Gem::Requirement.new(">= 2.3.0")
+
+  spec.metadata["allowed_push_host"] = "https://rubygems.org"
+
+  spec.metadata["homepage_uri"] = spec.homepage
+  spec.metadata["source_code_uri"] = "https://github.com/rich-bro/crawlab_ruby_sdk"
+  spec.metadata["changelog_uri"] = "https://github.com/rich-bro/crawlab_ruby_sdk"
+
+  # Specify which files should be added to the gem when it is released.
+  # The `git ls-files -z` loads the files in the RubyGem that have been added into git.
+  spec.files         = Dir.chdir(File.expand_path('..', __FILE__)) do
+    `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features)/}) }
+  end
+  spec.bindir        = "exe"
+  spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
+  spec.require_paths = ["lib"]
+end

data/lib/client/task_service_client.rb

@@ -0,0 +1,15 @@
+class TaskServiceClient
+	attr_accessor :address
+	attr_accessor :auth
+
+	def initialize(address,auth)
+		@address = address
+		@auth = auth
+	end
+
+    # 返回 TaskServiceSubscribeClient
+	def subscribe
+		stub = TaskServiceSubscribeClient.new(@address,@auth)
+		return stub
+	end
+end

data/lib/client/task_service_subscribe_client.rb

@@ -0,0 +1,52 @@
+class TaskServiceSubscribeClient
+	attr_accessor :stub
+	attr_accessor :metadata
+
+	def initialize(address,auth)
+		@stub = GRPC::ClientStub.new(address,:this_channel_is_insecure)
+		@metadata = {"authorization": auth}
+
+	end
+
+	def Send(streamMessage)
+		@stub.client_streamer("/grpc.TaskService/Subscribe", streamMessage, method(:marshal), method(:unmarshal),metadata: @metadata)
+	end
+
+
+	def marshal(obj)
+	# 	a = {
+	# 		code: obj.code,
+	# 		node_key: obj.node_key,
+	# 		key: obj.key,
+	# 		from: obj.from,
+	# 		to: obj.to,
+	# 		data: obj.data,
+	# 		error: obj.error,
+	# 	}
+
+       # return a.to_json
+       return Grpc::StreamMessage.encode(obj)
+	end
+
+	def unmarshal(str)
+		# a = JSON.parse(str)
+
+		# obj = StreamMessage.new(a)
+		# return obj
+
+		return Grpc::StreamMessage.decode(obj)
+	end
+
+	def each
+		
+	end
+
+	def test_send(streamMessage)
+		test_a(method(:marshal),streamMessage)
+	end
+
+	def test_a(bbb,streamMessage)
+		a = bbb.call(streamMessage)
+		puts a
+	end
+end

data/lib/crawlab_ruby_sdk/version.rb

@@ -0,0 +1,3 @@
+module CrawlabRubySdk
+  VERSION = "0.1.0"
+end

data/lib/crawlab_ruby_sdk.rb

@@ -0,0 +1,60 @@
+require "crawlab_ruby_sdk/version"
+require "grpc"
+require "json"
+def traverse_dir(file_path)
+  if File.directory? file_path
+    Dir.foreach(file_path){|file| traverse_dir(file_path+"/"+file) if file!="." and file!=".." }
+  else
+    load file_path if file_path.to_s.match(/\.rb$/)
+  end
+end
+
+PWD = File.expand_path("..", __FILE__)
+
+# puts PWD
+
+traverse_dir(PWD+'/entity/stream_message_code_pb.rb')
+traverse_dir(PWD+'/entity/result_pb.rb')
+traverse_dir(PWD+'/entity/stream_message_pb.rb')
+traverse_dir(PWD+'/entity/stream_message_data_task_pb.rb')
+traverse_dir(PWD+'/client')
+
+module CrawlabRubySdk
+  class Error < StandardError; end
+  # Your code goes here...
+
+
+  def self.save_item(item={})
+    address = ENV["CRAWLAB_GRPC_ADDRESS"]
+    if address==nil || address == ""
+      address = "localhost:9666"
+    end  
+
+    auth = ENV["CRAWLAB_GRPC_AUTH_KEY"]
+    if auth==nil || auth == ""
+      auth = "Crawlab2021!"
+    end  
+    
+    client = TaskServiceClient.new(address,auth)
+
+    sub_client = client.subscribe
+
+    task_id = ENV["CRAWLAB_TASK_ID"]
+    if task_id == nil || task_id == ""
+      task_id = "6486e256fa1cb07a47c09adf"
+    end
+    item = {"hello": "world","_tid":task_id}
+    item = Grpc::Result.new({"_tid":task_id,"name":"lala","age":"12"})
+
+    data = Grpc::StreamMessageDataTask.new()
+    # puts item
+
+    data.task_id = task_id
+    data.data.push(item)
+
+    puts data
+    msg = Grpc::StreamMessage.new(code:3,data:data.to_json)
+    puts msg
+    sub_client.Send([msg])    
+  end
+end

data/lib/entity/node_info_pb.rb

@@ -0,0 +1,17 @@
+# Generated by the protocol buffer compiler.  DO NOT EDIT!
+# source: entity/node_info.proto
+
+require 'google/protobuf'
+
+Google::Protobuf::DescriptorPool.generated_pool.build do
+  add_file("entity/node_info.proto", :syntax => :proto3) do
+    add_message "grpc.NodeInfo" do
+      optional :key, :string, 1
+      optional :is_master, :bool, 2
+    end
+  end
+end
+
+module Grpc
+  NodeInfo = ::Google::Protobuf::DescriptorPool.generated_pool.lookup("grpc.NodeInfo").msgclass
+end

data/lib/entity/plugin_request_pb.rb

@@ -0,0 +1,18 @@
+# Generated by the protocol buffer compiler.  DO NOT EDIT!
+# source: entity/plugin_request.proto
+
+require 'google/protobuf'
+
+Google::Protobuf::DescriptorPool.generated_pool.build do
+  add_file("entity/plugin_request.proto", :syntax => :proto3) do
+    add_message "grpc.PluginRequest" do
+      optional :name, :string, 1
+      optional :node_key, :string, 2
+      optional :data, :bytes, 3
+    end
+  end
+end
+
+module Grpc
+  PluginRequest = ::Google::Protobuf::DescriptorPool.generated_pool.lookup("grpc.PluginRequest").msgclass
+end

data/lib/entity/request_pb.rb

@@ -0,0 +1,17 @@
+# Generated by the protocol buffer compiler.  DO NOT EDIT!
+# source: entity/request.proto
+
+require 'google/protobuf'
+
+Google::Protobuf::DescriptorPool.generated_pool.build do
+  add_file("entity/request.proto", :syntax => :proto3) do
+    add_message "grpc.Request" do
+      optional :node_key, :string, 1
+      optional :data, :bytes, 2
+    end
+  end
+end
+
+module Grpc
+  Request = ::Google::Protobuf::DescriptorPool.generated_pool.lookup("grpc.Request").msgclass
+end

data/lib/entity/response_code_pb.rb

@@ -0,0 +1,15 @@
+# Generated by the protocol buffer compiler.  DO NOT EDIT!
+# source: entity/response_code.proto
+
+require 'google/protobuf'
+
+Google::Protobuf::DescriptorPool.generated_pool.build do
+  add_file("entity/response_code.proto", :syntax => :proto3) do
+    add_enum "ResponseCode" do
+      value :OK, 0
+      value :ERROR, 1
+    end
+  end
+end
+
+ResponseCode = ::Google::Protobuf::DescriptorPool.generated_pool.lookup("ResponseCode").enummodule

data/lib/entity/response_pb.rb

@@ -0,0 +1,22 @@
+# Generated by the protocol buffer compiler.  DO NOT EDIT!
+# source: entity/response.proto
+
+require 'google/protobuf'
+
+require 'entity/response_code_pb'
+
+Google::Protobuf::DescriptorPool.generated_pool.build do
+  add_file("entity/response.proto", :syntax => :proto3) do
+    add_message "grpc.Response" do
+      optional :code, :enum, 1, "ResponseCode"
+      optional :message, :string, 2
+      optional :data, :bytes, 3
+      optional :error, :string, 4
+      optional :total, :int64, 5
+    end
+  end
+end
+
+module Grpc
+  Response = ::Google::Protobuf::DescriptorPool.generated_pool.lookup("grpc.Response").msgclass
+end

data/lib/entity/result_pb.rb

@@ -0,0 +1,15 @@
+require 'google/protobuf'
+
+Google::Protobuf::DescriptorPool.generated_pool.build do
+  add_file("entity/result.proto", :syntax => :proto3) do
+    add_message "grpc.Result" do
+      optional :_tid, :string, 1
+      optional :name, :string, 2
+      optional :age, :string, 3
+    end
+  end
+end
+
+module Grpc
+  Result = ::Google::Protobuf::DescriptorPool.generated_pool.lookup("grpc.Result").msgclass
+end

data/lib/entity/stream_message_code_pb.rb

@@ -0,0 +1,28 @@
+# Generated by the protocol buffer compiler.  DO NOT EDIT!
+# source: entity/stream_message_code.proto
+
+require 'google/protobuf'
+
+Google::Protobuf::DescriptorPool.generated_pool.build do
+  add_file("proto/entity/stream_message_code.proto", :syntax => :proto3) do
+    add_enum "grpc.StreamMessageCode" do
+      value :PING, 0
+      value :RUN_TASK, 1
+      value :CANCEL_TASK, 2
+      value :INSERT_DATA, 3
+      value :INSERT_LOGS, 4
+      value :SEND_EVENT, 5
+      value :INSTALL_PLUGIN, 6
+      value :UNINSTALL_PLUGIN, 7
+      value :START_PLUGIN, 8
+      value :STOP_PLUGIN, 9
+      value :CONNECT, 10
+      value :DISCONNECT, 11
+      value :SEND, 12
+    end
+  end
+end
+
+module Grpc
+  StreamMessageCode = ::Google::Protobuf::DescriptorPool.generated_pool.lookup("grpc.StreamMessageCode").enummodule
+end

data/lib/entity/stream_message_data_task_pb.rb

@@ -0,0 +1,58 @@
+# # Generated by the protocol buffer compiler.  DO NOT EDIT!
+# # source: entity/stream_message_data_task.proto
+
+# require 'google/protobuf'
+
+# Google::Protobuf::DescriptorPool.generated_pool.build do
+#   add_file(PWD + "/proto/entity/stream_message_data_task.proto", :syntax => :proto3) do
+#     add_message "grpc.StreamMessageDataTask" do
+#       optional :task_id, :string, 1
+#       optional :data, :string,:repeated, 2
+#     end
+#   end
+# end
+
+# module Grpc
+#   StreamMessageDataTask = ::Google::Protobuf::DescriptorPool.generated_pool.lookup("grpc.StreamMessageDataTask").msgclass
+# end
+
+
+# frozen_string_literal: true
+# Generated by the protocol buffer compiler.  DO NOT EDIT!
+# source: entity/stream_message_data_task.proto
+
+require 'google/protobuf'
+
+# require 'entity/result_pb'
+
+
+descriptor_data = "\n%entity/stream_message_data_task.proto\x12\x04grpc\x1a\x13\x65ntity/result.proto\"D\n\x15StreamMessageDataTask\x12\x0f\n\x07task_id\x18\x01 \x01(\t\x12\x1a\n\x04\x64\x61ta\x18\x02 \x03(\x0b\x32\x0c.grpc.ResultB\x08Z\x06.;grpcb\x06proto3"
+
+pool = Google::Protobuf::DescriptorPool.generated_pool
+
+begin
+  pool.add_serialized_file(descriptor_data)
+rescue TypeError => e
+  # Compatibility code: will be removed in the next major version.
+  require 'google/protobuf/descriptor_pb'
+  parsed = Google::Protobuf::FileDescriptorProto.decode(descriptor_data)
+  parsed.clear_dependency
+  serialized = parsed.class.encode(parsed)
+  file = pool.add_serialized_file(serialized)
+  warn "Warning: Protobuf detected an import path issue while loading generated file #{__FILE__}"
+  imports = [
+    ["grpc.Result", "entity/result.proto"],
+  ]
+  imports.each do |type_name, expected_filename|
+    import_file = pool.lookup(type_name).file_descriptor
+    if import_file.name != expected_filename
+      warn "- #{file.name} imports #{expected_filename}, but that import was loaded as #{import_file.name}"
+    end
+  end
+  warn "Each proto file must use a consistent fully-qualified name."
+  warn "This will become an error in the next major version."
+end
+
+module Grpc
+  StreamMessageDataTask = ::Google::Protobuf::DescriptorPool.generated_pool.lookup("grpc.StreamMessageDataTask").msgclass
+end

data/lib/entity/stream_message_pb.rb

@@ -0,0 +1,24 @@
+# Generated by the protocol buffer compiler.  DO NOT EDIT!
+# source: entity/stream_message.proto
+
+require 'google/protobuf'
+
+# require 'entity/stream_message_code_pb'
+
+Google::Protobuf::DescriptorPool.generated_pool.build do
+  add_file("proto/entity/stream_message.proto", :syntax => :proto3) do
+    add_message "grpc.StreamMessage" do
+      optional :code, :enum, 1, "grpc.StreamMessageCode"
+      optional :node_key, :string, 2
+      optional :key, :string, 3
+      optional :from, :string, 4
+      optional :to, :string, 5
+      optional :data, :bytes, 6
+      optional :error, :string, 7
+    end
+  end
+end
+
+module Grpc
+  StreamMessage = ::Google::Protobuf::DescriptorPool.generated_pool.lookup("grpc.StreamMessage").msgclass
+end

data/lib/models/node_pb.rb

@@ -0,0 +1,27 @@
+# Generated by the protocol buffer compiler.  DO NOT EDIT!
+# source: models/node.proto
+
+require 'google/protobuf'
+
+Google::Protobuf::DescriptorPool.generated_pool.build do
+  add_file("proto/models/node.proto", :syntax => :proto3) do
+    add_message "grpc.Node" do
+      optional :_id, :string, 1
+      optional :name, :string, 2
+      optional :ip, :string, 3
+      optional :port, :string, 5
+      optional :mac, :string, 6
+      optional :hostname, :string, 7
+      optional :description, :string, 8
+      optional :key, :string, 9
+      optional :is_master, :bool, 11
+      optional :update_ts, :string, 12
+      optional :create_ts, :string, 13
+      optional :update_ts_unix, :int64, 14
+    end
+  end
+end
+
+module Grpc
+  Node = ::Google::Protobuf::DescriptorPool.generated_pool.lookup("grpc.Node").msgclass
+end

data/lib/models/task_pb.rb

@@ -0,0 +1,26 @@
+# Generated by the protocol buffer compiler.  DO NOT EDIT!
+# source: models/task.proto
+
+require 'google/protobuf'
+
+Google::Protobuf::DescriptorPool.generated_pool.build do
+  add_file("models/task.proto", :syntax => :proto3) do
+    add_message "grpc.Task" do
+      optional :_id, :string, 1
+      optional :spider_id, :string, 2
+      optional :status, :string, 5
+      optional :node_id, :string, 6
+      optional :cmd, :string, 8
+      optional :param, :string, 9
+      optional :error, :string, 10
+      optional :pid, :int32, 16
+      optional :run_type, :string, 17
+      optional :schedule_id, :string, 18
+      optional :type, :string, 19
+    end
+  end
+end
+
+module Grpc
+  Task = ::Google::Protobuf::DescriptorPool.generated_pool.lookup("grpc.Task").msgclass
+end

metadata

@@ -0,0 +1,68 @@
+--- !ruby/object:Gem::Specification
+name: crawlab_ruby_sdk
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- min
+autorequire:
+bindir: exe
+cert_chain: []
+date: 2023-06-14 00:00:00.000000000 Z
+dependencies: []
+description: This guy is very lazy.
+email:
+- lijinmin3903@126.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- Gemfile
+- README.md
+- Rakefile
+- bin/console
+- bin/setup
+- crawlab_ruby_sdk.gemspec
+- lib/client/task_service_client.rb
+- lib/client/task_service_subscribe_client.rb
+- lib/crawlab_ruby_sdk.rb
+- lib/crawlab_ruby_sdk/version.rb
+- lib/entity/node_info_pb.rb
+- lib/entity/plugin_request_pb.rb
+- lib/entity/request_pb.rb
+- lib/entity/response_code_pb.rb
+- lib/entity/response_pb.rb
+- lib/entity/result_pb.rb
+- lib/entity/stream_message_code_pb.rb
+- lib/entity/stream_message_data_task_pb.rb
+- lib/entity/stream_message_pb.rb
+- lib/models/node_pb.rb
+- lib/models/task_pb.rb
+homepage: https://github.com/rich-bro/crawlab_ruby_sdk
+licenses: []
+metadata:
+  allowed_push_host: https://rubygems.org
+  homepage_uri: https://github.com/rich-bro/crawlab_ruby_sdk
+  source_code_uri: https://github.com/rich-bro/crawlab_ruby_sdk
+  changelog_uri: https://github.com/rich-bro/crawlab_ruby_sdk
+post_install_message:
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: 2.3.0
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubygems_version: 3.1.4
+signing_key:
+specification_version: 4
+summary: This guy is very lazy.
+test_files: []