crawlab_ruby_sdk 0.1.7 → 0.1.8

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 1c85282caf96edf6a9100a70b045bb1004a471db9d2a7842f83e77a28fbde32e
-  data.tar.gz: cc64ce0004eeede08da51a5a7eec294f0db1b494d897079562a1578128960a1e
+  metadata.gz: c5c857dd2f243d47318786c542faec1a9fc7a6a52bba0edf7523313379b4f2d4
+  data.tar.gz: fbbacb701ad2bd5d090d85ba3322cd82a5c02f5d7d0d3ab729ee2b2b0d0882c1
 SHA512:
-  metadata.gz: 012211e75c7de44e0493c931ce3d6d4cd8f1351da52ae4bd77da3aca95e39294d3c4c9e2c5cf2d5ebd877dcb1dbd2e45bc9f9b8ce2199ffdf3d71c3f6de517a4
-  data.tar.gz: 380d04dddd44c608e80eebc23f601f23acaacc08fe291e6a711749e65b00d7aef664a8d30a0a0537bfec82c4e02620da3dfdbcf259bd1bd6aecc2bebaaf821d2
+  metadata.gz: 0f9752ef0ad2e59d72fa870a6493a8d831716c0422bb4d6d08a0afc3cadc08ee9b4b80eb97af5192ab1094ad7884533b3c7c75cc76d1a47fe7c0b90f991538d0
+  data.tar.gz: 47e6a0373338ad0d32b8286d2b422da55a67d5728655fca51a028a17928ee301b981d17cb396e06d23f8911ac37663b471360f8ecce97e55b7f8595e4e8096c1

data/README.md

@@ -39,6 +39,19 @@ Or install it yourself as:
     puts bucket_url 
 ```
 
+##### 字段验证规则
+
+```
+ CrawlabRubySdk.save_item({name: "haha",age:12},table_name)
+ CrawlabRubySdk.save_items([{name: "haha",age:12},{name:"456",age:34}],table_name)
+ table_name 可以是这几个["thinktank_expert_reports", "thinktank_informations", "thinktank_experts", "thinktank_reports"] 中的一个,也可以不传递
+
+ 1、如果当前采集的结果 存储的数据表是上面几个中的一个，无论是否传递，都会按照对应的表结构的 字段规则验证字段
+ 2、如果当前采集的结果，存储的数据表不属于上面几个中的一个，则按照传递传递的table_name，对应的字段规则验证字段
+ 3、如果没有传递table_name,并且采集的结果存储临时表，则根据推送的结果的字段匹配 上面四个表的字段，匹配上了就按照匹配上的表结构的字段规则验证，没有匹配上则不验证
+```
+
+
 ## Development
 
 After checking out the repo, run `bin/setup` to install dependencies. You can also run `bin/console` for an interactive prompt that will allow you to experiment.

data/lib/crawlab_ruby_sdk/version.rb

@@ -1,3 +1,3 @@
 module CrawlabRubySdk
-  VERSION = "0.1.7"
+  VERSION = "0.1.8"
 end

data/lib/crawlab_ruby_sdk.rb

@@ -36,6 +36,7 @@ module CrawlabRubySdk
       auth = "Crawlab2021!"
     end  
 
+    table_name = get_table_name(table_name)
     if !Verify.IsVerified?([item],table_name)
       return
     end
@@ -57,7 +58,9 @@ module CrawlabRubySdk
     if auth==nil || auth == ""
       auth = "Crawlab2021!"
     end  
-    if !Verify.IsVerified?([item],table_name)
+
+    table_name = get_table_name(table_name)
+    if !Verify.IsVerified?(items,table_name)
       return
     end    
     
@@ -99,10 +102,11 @@ module CrawlabRubySdk
 
     data = {task_id: task_id,data:records}.to_json.b
 
+    # puts "=====#{{task_id: task_id,data:records}.to_json}==="
+
     msg = Grpc::StreamMessage.new(code:3,data:data)
-    puts data
 
-    # sub_client.Send([msg]) 
+    sub_client.Send([msg]) 
   end
 
   def self.get_task_id
@@ -120,4 +124,14 @@ module CrawlabRubySdk
   def self.save_file_stream_to_oss(oss_path,stream)
     OssServerClient.new.send_stream(oss_path,stream)
   end
+
+  def self.get_table_name(table_name="")
+    table_names = Verify.table_names
+    if ENV["TABLE_NAME"] != nil && ENV["TABLE_NAME"] != "" && table_names.include?(ENV["TABLE_NAME"])
+      table_name = ENV["TABLE_NAME"]
+      return table_name
+    end 
+
+    return table_name
+  end
 end

data/lib/models/base.rb

@@ -85,6 +85,7 @@ class BaseModel
 		rescue StandardError => e
 			return false
 		end
+		return true
 	end
 
 	def verify_string(v)
@@ -161,7 +162,7 @@ class BaseModel
 			return false
 		end		
 		fields_arr = fn.split(":")
-		if length_arr.size < 2
+		if fields_arr.size < 2
 			return false
 		end
 		fields_str = fields_arr[1]
@@ -169,15 +170,17 @@ class BaseModel
 		fields = fields_str.split(",")
 
 		begin
-			datas = json.parse(v)
+			datas = JSON.parse(v)
 			datas.each do |data|
 				fields.each do |field|
 					if data[field] == nil
+						puts "ERROR #{field} not Exist!"
 						return false
 					end
 				end
 			end
 		rescue StandardError => e
+			puts e
 			return false
 		end
 

data/lib/models/thinktank_expert.rb

@@ -41,7 +41,40 @@ class ThinktankExpert < BaseModel
 	
 
 	def initialize(options={})
-		
+		options = JSON.parse(options.to_json)
+		@id = options["id"]
+		@name = options["name"]
+		@title = options["title"]
+		@content = options["content"]
+		@location = options["location"]
+		@area_of_expertise = options["area_of_expertise"]
+		@profile_images = options["profile_images"]
+		@phone = options["phone"]
+		@email = options["email"]
+		@link = options["link"]
+		@audios = options["audios"]
+		@videos = options["videos"]
+		@education = options["education"]
+		@related_topics = options["related_topics"]
+		@site_name = options["site_name"]
+		@site_name_cn = options["site_name_cn"]
+		@domain = options["domain"]
+		@created_at = options["created_at"]
+		@updated_at = options["updated_at"]
+		@source = options["source"]
+		@oss_profile_images = options["oss_profile_images"]
+		@facebook = options["facebook"]
+		@twitter = options["twitter"]
+		@linkedin = options["linkedin"]
+		@instagram = options["instagram"]
+		@wikidata = options["wikidata"]
+		@person_type = options["person_type"]
+		@files = options["files"]
+		@oss_files = options["oss_files"]
+		@associated_program = options["associated_program"]
+		@lang = options["lang"]
+		@website = options["website"]
+		@nationalities = options["nationalities"]
 	end
 
 	def as_json

data/lib/models/thinktank_expert_report.rb

@@ -42,7 +42,43 @@ class ThinktankExpertReport < BaseModel
 	register
 
 	def initialize(options={})
-		
+		options = JSON.parse(options.to_json)
+		@id = options["id"]
+		@title = options["title"]
+		@site_name = options["site_name"]
+		@site_name_cn = options["site_name_cn"]
+		@abstract = options["abstract"]
+		@content = options["content"]
+		@author_names = options["author_names"]
+		@source = options["source"]
+		@files = options["files"]
+		@images = options["images"]
+		@videos = options["videos"]
+		@audios = options["audios"]
+		@links = options["links"]
+		@domain = options["domain"]
+		@keywords= options["keywords"]
+		@html_content = options["html_content"]
+		@lang = options["lang"]
+		@country_cn = options["country_cn"]
+		@country_code = options["country_code"]
+		@created_at = options["created_at"]
+		@updated_at = options["updated_at"]
+		@created_time = options["created_time"]
+		@oss_files = options["oss_files"]
+		@oss_images = options["oss_images"]
+		@customer_category = options["customer_category"]
+		@category = options["category"]
+		@topics = options["topics"]
+		@tags = options["tags"]
+		@views = options["views"]
+		@comments = options["comments"]
+		@reference = options["reference"]
+		@mention_country = options["mention_country"]
+		@authors = options["authors"]
+		@sub_title = options["sub_title"]
+		@timezone = options["timezone"]
+		@timezone_location = options["timezone_location"]
 	end
 
 	def as_json

data/lib/models/thinktank_information.rb

@@ -44,7 +44,44 @@ class ThinktankInformation < BaseModel
 
 
 	def initialize(options={})
-		
+		options = JSON.parse(options.to_json)
+		@id = options["id"]
+		@title = options["title"]
+		@site_name = options["site_name"]
+		@site_name_cn = options["site_name_cn"]
+		@abstract = options["abstract"]
+		@content = options["content"]
+		@author_names = options["author_names"]
+		@state_info = options["state_info"]
+		@source = options["source"]
+		@files = options["files"]
+		@images = options["images"]
+		@videos = options["videos"]
+		@audios = options["audios"]
+		@links = options["links"]
+		@domain = options["domain"]
+		@keywords = options["keywords"]
+		@html_content = options["html_content"]
+		@lang = options["lang"]
+		@country_cn = options["country_cn"]
+		@country_code = options["country_code"]
+		@created_at = options["created_at"]
+		@updated_at = options["updated_at"]
+		@created_time = options["created_time"]
+		@oss_files = options["oss_files"]
+		@oss_images = options["oss_images"]
+		@customer_category = options["customer_category"]
+		@category = options["category"]
+		@topics = options["topics"]
+		@tags = options["tags"]
+		@views = options["views"]
+		@comments = options["comments"]
+		@reference = options["reference"]
+		@mention_country = options["mention_country"]
+		@authors = options["authors"]
+		@sub_title = options["sub_title"]
+		@timezone = options["timezone"]
+		@timezone_location = options["timezone_location"]
 	end
 
 	def as_json

data/lib/models/thinktank_report.rb

@@ -43,7 +43,43 @@ class ThinktankReport < BaseModel
 
 
 	def initialize(options = {})
-		
+		options = JSON.parse(options.to_json)
+		@id = options["id"]
+		@title = options["title"]
+		@site_name = options["site_name"]
+		@site_name_cn = options["site_name_cn"]
+		@abstract = options["abstract"]
+		@content = options["content"]
+		@author_names = options["author_names"]
+		@source = options["source"]
+		@files = options["files"]
+		@images = options["images"]
+		@videos = options["videos"]
+		@audios = options["audios"]
+		@links = options["links"]
+		@domain = options["domain"]
+		@keywords = options["keywords"]
+		@html_content = options["html_content"]
+		@lang = options["lang"]
+		@country_cn = options["country_cn"]
+		@country_code = options["country_code"]
+		@created_at = options["created_at"]
+		@updated_at = options["updated_at"]
+		@created_time = options["created_time"]
+		@oss_files = options["oss_files"]
+		@oss_images = options["oss_images"]
+		@customer_category = options["customer_category"]
+		@category = options["category"]
+		@topics = options["topics"]
+		@tags = options["tags"]
+		@views = options["views"]
+		@comments = options["comments"]
+		@reference = options["reference"]
+		@mention_country = options["mention_country"]
+		@authors = options["authors"]
+		@sub_title = options["sub_title"]
+		@timezone = options["timezone"]
+		@timezone_location = options["timezone_location"]
 	end
 
 	def to_json

data/lib/verify/verify.rb

@@ -3,6 +3,10 @@ class Verify
 
 		if table_name == "" && items.size > 0
 			table_name = switch_table(items[0])
+		elsif table_name != "" && items.size > 0 
+			if !match_fields?(items[0],table_name)
+				return
+			end
 		end
 
 		if table_name == ""
@@ -50,4 +54,40 @@ class Verify
 		return table_name
 	end
 
+	def self.match_fields?(item={},table_name)
+		if table_name == nil || table_name == ""
+			return false
+		end
+		item_keys = JSON.parse(item.to_json).keys
+		v = $map_models[table_name]
+
+		if v == nil 
+			puts "ERROR #{table_name} not Exist!"
+			return false
+		end
+
+		klass = Object.const_get v
+		klass_keys = klass.fields
+
+		# puts "----#{klass_keys}--"
+		sub_keys1 = item_keys - klass_keys
+		if sub_keys1.size > 0
+			puts "ERROR #{sub_keys1} do not belong #{table_name}"
+			return false
+		end
+
+		sub_keys2 = klass_keys - item_keys
+		if sub_keys2.size > 0
+			puts "ERROR #{sub_keys2} do not Exist!"
+			return false
+		end
+
+		return true
+
+	end
+
+	def self.table_names
+		return $map_models.keys
+	end
+
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: crawlab_ruby_sdk
 version: !ruby/object:Gem::Version
-  version: 0.1.7
+  version: 0.1.8
 platform: ruby
 authors:
 - min
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2023-07-10 00:00:00.000000000 Z
+date: 2023-07-11 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: grpc