crawlab_ruby_sdk 0.1.6 → 0.1.8

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 3fc9c6d1277660d63c17811f472dd56d438a4998c6d0c03e99d1a9e5c0357868
-  data.tar.gz: 20b8ffa1ba0b1275c66f1a79cc80a479d39211b781174107a4cc21bf8ca86fb1
+  metadata.gz: c5c857dd2f243d47318786c542faec1a9fc7a6a52bba0edf7523313379b4f2d4
+  data.tar.gz: fbbacb701ad2bd5d090d85ba3322cd82a5c02f5d7d0d3ab729ee2b2b0d0882c1
 SHA512:
-  metadata.gz: 5b925b182c7fa7fcbd8d7c9d3a2883e8acee1fa41eafc01a28aaa1ba78c037f8d17f1fed831846001b1761b1480854e908732ac5e8e3b500828267baebc1721c
-  data.tar.gz: ae63a9d061c104cf8ad8c3e9c4d72b1a70f9f21dbe4c7b07d299ee5cfafe048a008f16ee26ed94efda8b64542aed817f33c2dbdd0323cab28d0c2ce1d8cd67ea
+  metadata.gz: 0f9752ef0ad2e59d72fa870a6493a8d831716c0422bb4d6d08a0afc3cadc08ee9b4b80eb97af5192ab1094ad7884533b3c7c75cc76d1a47fe7c0b90f991538d0
+  data.tar.gz: 47e6a0373338ad0d32b8286d2b422da55a67d5728655fca51a028a17928ee301b981d17cb396e06d23f8911ac37663b471360f8ecce97e55b7f8595e4e8096c1

data/README.md

@@ -39,6 +39,19 @@ Or install it yourself as:
     puts bucket_url 
 ```
 
+##### 字段验证规则
+
+```
+ CrawlabRubySdk.save_item({name: "haha",age:12},table_name)
+ CrawlabRubySdk.save_items([{name: "haha",age:12},{name:"456",age:34}],table_name)
+ table_name 可以是这几个["thinktank_expert_reports", "thinktank_informations", "thinktank_experts", "thinktank_reports"] 中的一个,也可以不传递
+
+ 1、如果当前采集的结果 存储的数据表是上面几个中的一个，无论是否传递，都会按照对应的表结构的 字段规则验证字段
+ 2、如果当前采集的结果，存储的数据表不属于上面几个中的一个，则按照传递传递的table_name，对应的字段规则验证字段
+ 3、如果没有传递table_name,并且采集的结果存储临时表，则根据推送的结果的字段匹配 上面四个表的字段，匹配上了就按照匹配上的表结构的字段规则验证，没有匹配上则不验证
+```
+
+
 ## Development
 
 After checking out the repo, run `bin/setup` to install dependencies. You can also run `bin/console` for an interactive prompt that will allow you to experiment.

data/lib/crawlab_ruby_sdk/version.rb

@@ -1,3 +1,3 @@
 module CrawlabRubySdk
-  VERSION = "0.1.6"
+  VERSION = "0.1.8"
 end

data/lib/crawlab_ruby_sdk.rb

@@ -1,4 +1,4 @@
-require "crawlab_ruby_sdk/version"
+# require "crawlab_ruby_sdk/version"
 require "grpc"
 require "json"
 def traverse_dir(file_path)
@@ -14,17 +14,18 @@ dir = File.expand_path("..", __FILE__)
 # puts dir
 
 traverse_dir(dir+'/entity/stream_message_code_pb.rb')
-traverse_dir(dir+'/entity/result_pb.rb')
 traverse_dir(dir+'/entity/stream_message_pb.rb')
-traverse_dir(dir+'/entity/stream_message_data_task_pb.rb')
 traverse_dir(dir+'/client')
+traverse_dir(dir+'/models/base.rb')
+traverse_dir(dir+'/models')
+traverse_dir(dir+'/verify')
 
 module CrawlabRubySdk
   class Error < StandardError; end
   # Your code goes here...
 
   
-  def self.save_item(item={})
+  def self.save_item(item={},table_name="")
     address = ENV["CRAWLAB_GRPC_ADDRESS"]
     if address==nil || address == ""
       address = "localhost:9666"
@@ -34,6 +35,11 @@ module CrawlabRubySdk
     if auth==nil || auth == ""
       auth = "Crawlab2021!"
     end  
+
+    table_name = get_table_name(table_name)
+    if !Verify.IsVerified?([item],table_name)
+      return
+    end
     
     client = TaskServiceClient.new(address,auth)
 
@@ -42,7 +48,7 @@ module CrawlabRubySdk
     save(sub_client,[item])
   end
 
-  def self.save_items(items=[])
+  def self.save_items(items=[],table_name="")
     address = ENV["CRAWLAB_GRPC_ADDRESS"]
     if address==nil || address == ""
       address = "localhost:9666"
@@ -52,6 +58,11 @@ module CrawlabRubySdk
     if auth==nil || auth == ""
       auth = "Crawlab2021!"
     end  
+
+    table_name = get_table_name(table_name)
+    if !Verify.IsVerified?(items,table_name)
+      return
+    end    
     
     client = TaskServiceClient.new(address,auth)
 
@@ -90,11 +101,8 @@ module CrawlabRubySdk
     end
 
     data = {task_id: task_id,data:records}.to_json.b
-    # data = data.encode("utf-8")
-    # puts data
 
-    # data = data.encode('ASCII-8BIT', invalid: :replace, undef: :replace, replace: '')
-    # puts data
+    # puts "=====#{{task_id: task_id,data:records}.to_json}==="
 
     msg = Grpc::StreamMessage.new(code:3,data:data)
 
@@ -116,4 +124,14 @@ module CrawlabRubySdk
   def self.save_file_stream_to_oss(oss_path,stream)
     OssServerClient.new.send_stream(oss_path,stream)
   end
+
+  def self.get_table_name(table_name="")
+    table_names = Verify.table_names
+    if ENV["TABLE_NAME"] != nil && ENV["TABLE_NAME"] != "" && table_names.include?(ENV["TABLE_NAME"])
+      table_name = ENV["TABLE_NAME"]
+      return table_name
+    end 
+
+    return table_name
+  end
 end

data/lib/models/base.rb

@@ -0,0 +1,191 @@
+require 'json'
+
+$map_models = {}
+class BaseModel
+	def self.table_name
+		return ""
+	end
+
+	def self.register
+		$map_models[table_name]  = self.name
+	end
+
+	def self.fields
+		JSON.parse(self.new({}).as_json.to_json).keys
+	end
+
+	def self.verify_keys
+		return {}
+	end
+
+	def as_json
+		{}
+	end
+
+	def verify
+		json_data = as_json
+		verify_keys = self.class.verify_keys
+
+		verify_keys.each do |key,fns|
+			value = json_data[key.to_sym]
+			if fns.class.name == "Array"
+				fns.each do |fn|
+					if fn == "empty"
+						if !verify_empty(value)
+							puts "ERROR: #{key} cannot be empty!"
+							return false
+						end
+					elsif fn == "json"
+						if !verify_json(value)
+							puts "ERROR: #{key} json string parse fail!"
+							return false
+						end
+					elsif fn == "string"
+						if !verify_string(value)
+							puts "ERROR: #{key} field type is not string!"
+							return false
+						end
+					elsif fn == "int"
+						if !verify_int(value)
+							puts "ERROR: #{key} field type is not int!"
+							return false
+						end
+					elsif fn.include?("regex")
+						if !verify_regex(fn,value)
+							puts "ERROR: #{key} regex #{fn} match error"
+							return false
+						end
+					elsif fn.include?("length")
+						if !verify_length(fn,value)
+							puts "ERROR: #{key} length must be #{fn}"
+							return false
+						end
+					elsif fn.include?("fields")
+						if !verify_fields(fn,value)
+							puts "ERROR: #{key}:#{fn} not Exist!"
+							return false
+						end
+					end
+				end
+			end
+		end
+	end
+
+	def verify_empty(v)
+		if v == nil || v == "" 
+			return false
+		end
+		return true
+	end
+
+	def verify_json(v)
+		begin
+			JSON.parse(v)
+			return true
+		rescue StandardError => e
+			return false
+		end
+		return true
+	end
+
+	def verify_string(v)
+		if v == nil
+			return false
+		end
+		if v.class.name == "String"
+			return true
+		end
+		return false
+	end
+
+	def verify_int(v)
+		if v == nil
+			return false
+		end
+
+		if  v.class.name == "Integer"
+			return true
+		end
+		return false
+	end
+
+	def verify_regex(fn,v)
+		if fn.class.name != "String"
+			return false
+		end		
+		regex_arr = fn.split(":")
+		if regex_arr.size < 2
+			return false
+		end
+		regex_str = regex_arr[1]
+		if regex_str.size == 0
+			return false
+		end
+
+		if v == nil || v == ""
+			return false
+		end
+
+		if v =~ Regexp.new(regex_str)
+		  return true
+		else
+			return false
+		end
+	end
+
+	def verify_length(fn,v)
+		if fn.class.name != "String"
+			return false
+		end		
+		length_arr = fn.split(":")
+		if length_arr.size < 2
+			return false
+		end
+		length = length_arr[1].to_i
+		if length == 0
+			return false
+		end		
+
+		if v == nil
+			return false
+		end
+
+		if v.to_s.size == length
+			return true
+		end
+
+		return false
+	end
+
+	def verify_fields(fn,v)
+		if fn.class.name != "String"
+			return false
+		end		
+		fields_arr = fn.split(":")
+		if fields_arr.size < 2
+			return false
+		end
+		fields_str = fields_arr[1]
+
+		fields = fields_str.split(",")
+
+		begin
+			datas = JSON.parse(v)
+			datas.each do |data|
+				fields.each do |field|
+					if data[field] == nil
+						puts "ERROR #{field} not Exist!"
+						return false
+					end
+				end
+			end
+		rescue StandardError => e
+			puts e
+			return false
+		end
+
+		return true
+	end
+
+
+end

data/lib/models/thinktank_expert.rb

@@ -0,0 +1,140 @@
+class ThinktankExpert < BaseModel
+	attr_accessor :id
+	attr_accessor :name
+	attr_accessor :title
+	attr_accessor :content
+	attr_accessor :location
+	attr_accessor :area_of_expertise
+	attr_accessor :profile_images
+	attr_accessor :phone
+	attr_accessor :email
+	attr_accessor :link
+	attr_accessor :audios
+	attr_accessor :videos
+	attr_accessor :education
+	attr_accessor :related_topics
+	attr_accessor :site_name
+	attr_accessor :site_name_cn
+	attr_accessor :domain
+	attr_accessor :created_at
+	attr_accessor :updated_at
+	attr_accessor :source
+	attr_accessor :oss_profile_images
+	attr_accessor :facebook
+	attr_accessor :twitter
+	attr_accessor :linkedin
+	attr_accessor :instagram
+	attr_accessor :wikidata
+	attr_accessor :person_type
+	attr_accessor :files
+	attr_accessor :oss_files
+	attr_accessor :associated_program
+	attr_accessor :lang
+	attr_accessor :website
+	attr_accessor :nationalities
+
+
+	def self.table_name
+		return "thinktank_experts"
+	end	
+	register
+	
+
+	def initialize(options={})
+		options = JSON.parse(options.to_json)
+		@id = options["id"]
+		@name = options["name"]
+		@title = options["title"]
+		@content = options["content"]
+		@location = options["location"]
+		@area_of_expertise = options["area_of_expertise"]
+		@profile_images = options["profile_images"]
+		@phone = options["phone"]
+		@email = options["email"]
+		@link = options["link"]
+		@audios = options["audios"]
+		@videos = options["videos"]
+		@education = options["education"]
+		@related_topics = options["related_topics"]
+		@site_name = options["site_name"]
+		@site_name_cn = options["site_name_cn"]
+		@domain = options["domain"]
+		@created_at = options["created_at"]
+		@updated_at = options["updated_at"]
+		@source = options["source"]
+		@oss_profile_images = options["oss_profile_images"]
+		@facebook = options["facebook"]
+		@twitter = options["twitter"]
+		@linkedin = options["linkedin"]
+		@instagram = options["instagram"]
+		@wikidata = options["wikidata"]
+		@person_type = options["person_type"]
+		@files = options["files"]
+		@oss_files = options["oss_files"]
+		@associated_program = options["associated_program"]
+		@lang = options["lang"]
+		@website = options["website"]
+		@nationalities = options["nationalities"]
+	end
+
+	def as_json
+		return {
+			id: @id,
+			name: @name,
+			title: @title,
+			content: @content,
+			location: @location,
+			area_of_expertise: @area_of_expertise,
+			profile_images: @profile_images,
+			phone: @phone,
+			email: @email,
+			link: @link,
+			audios: @audios,
+			videos: @videos,
+			education: @education,
+			related_topics: @related_topics,
+			site_name: @site_name,
+			site_name_cn: @site_name_cn,
+			domain: @domain,
+			created_at: @created_at,
+			updated_at: @updated_at,
+			source: @source,
+			oss_profile_images: @oss_profile_images,
+			facebook: @facebook,
+			twitter: @twitter,
+			linkedin: @linkedin,
+			instagram: @instagram,
+			wikidata: @wikidata,
+			person_type: @person_type,
+			files: @files,
+			oss_files: @oss_files,
+			associated_program: @associated_program,
+			lang: @lang,
+			website: @website,
+			nationalities: @nationalities,
+		}
+	end
+
+	def to_json
+		return as_json.to_json
+	end
+
+	def self.verify_keys
+		return {
+			"id"=>          ["empty", "string"],
+			"title"=>       ["json", "string"],
+			"name"=>        ["empty", "string"],
+			"site_name_cn"=>["empty", "string"],
+			"site_name"=>   ["empty", "string"],
+			"source"=>      ["empty", "string"],
+			"audios"=>      ["json", "string"],
+			"videos"=>      ["json", "string"],
+			"related_topics"=>["json", "string"],
+			"files"=>       ["json", "string"],
+			"oss_files"=>   ["json", "string"],
+			"domain"=>      ["empty", "string"],
+			"created_at"=>  ["empty", "int", "length:13"],
+			"updated_at"=>  ["empty", "int", "length:13"],			
+		}
+	end
+end

data/lib/models/thinktank_expert_report.rb

@@ -0,0 +1,160 @@
+class ThinktankExpertReport < BaseModel
+	attr_accessor :id
+	attr_accessor :title
+	attr_accessor :site_name
+	attr_accessor :site_name_cn
+	attr_accessor :abstract
+	attr_accessor :content
+	attr_accessor :author_names
+	attr_accessor :source
+	attr_accessor :files
+	attr_accessor :images
+	attr_accessor :videos
+	attr_accessor :audios
+	attr_accessor :links
+	attr_accessor :domain
+	attr_accessor :keywords
+	attr_accessor :html_content
+	attr_accessor :lang
+	attr_accessor :country_cn
+	attr_accessor :country_code
+	attr_accessor :created_at
+	attr_accessor :updated_at
+	attr_accessor :created_time
+	attr_accessor :oss_files
+	attr_accessor :oss_images
+	attr_accessor :customer_category
+	attr_accessor :category
+	attr_accessor :topics
+	attr_accessor :tags
+	attr_accessor :views
+	attr_accessor :comments
+	attr_accessor :reference
+	attr_accessor :mention_country
+	attr_accessor :authors
+	attr_accessor :sub_title
+	attr_accessor :timezone
+	attr_accessor :timezone_location
+
+	def self.table_name
+		return "thinktank_expert_reports"
+	end
+	register
+
+	def initialize(options={})
+		options = JSON.parse(options.to_json)
+		@id = options["id"]
+		@title = options["title"]
+		@site_name = options["site_name"]
+		@site_name_cn = options["site_name_cn"]
+		@abstract = options["abstract"]
+		@content = options["content"]
+		@author_names = options["author_names"]
+		@source = options["source"]
+		@files = options["files"]
+		@images = options["images"]
+		@videos = options["videos"]
+		@audios = options["audios"]
+		@links = options["links"]
+		@domain = options["domain"]
+		@keywords= options["keywords"]
+		@html_content = options["html_content"]
+		@lang = options["lang"]
+		@country_cn = options["country_cn"]
+		@country_code = options["country_code"]
+		@created_at = options["created_at"]
+		@updated_at = options["updated_at"]
+		@created_time = options["created_time"]
+		@oss_files = options["oss_files"]
+		@oss_images = options["oss_images"]
+		@customer_category = options["customer_category"]
+		@category = options["category"]
+		@topics = options["topics"]
+		@tags = options["tags"]
+		@views = options["views"]
+		@comments = options["comments"]
+		@reference = options["reference"]
+		@mention_country = options["mention_country"]
+		@authors = options["authors"]
+		@sub_title = options["sub_title"]
+		@timezone = options["timezone"]
+		@timezone_location = options["timezone_location"]
+	end
+
+	def as_json
+		return {
+			id: @id,
+			title: @title,
+			site_name: @site_name,
+			site_name_cn: @site_name_cn,
+			abstract: @abstract,
+			content: @content,
+			author_names: @author_names,
+			source: @source,
+			files: @files,
+			images: @images,
+			videos: @videos,
+			audios: @audios,
+			links: @links,
+			domain: @domain,
+			keywords: @keywords,
+			html_content: @html_content,
+			lang: @lang,
+			country_cn: @country_cn,
+			country_code: @country_code,
+			created_at: @created_at,
+			updated_at: @updated_at,
+			created_time: @created_time,
+			oss_files: @oss_files,
+			oss_images: @oss_images,
+			customer_category: @customer_category,
+			category: @category,
+			topics: @topics,
+			tags: @tags,
+			views: @views,
+			comments: @comments,
+			reference: @reference,
+			mention_country: @mention_country,
+			authors: @authors,
+			sub_title: @sub_title,
+			timezone: @timezone,
+			timezone_location: @timezone_location,
+		}
+	end
+
+	def to_json
+		return as_json.to_json
+	end
+
+	def self.verify_keys
+		return {
+			"id"=>            ["empty", "string"],
+			"title"=>         ["empty", "string"],
+			"site_name"=>     ["empty", "string"],
+			"site_name_cn"=>  ["empty", "string"],
+			"content"=>       ["empty", "string"],
+			"source"=>        ["empty", "string"],
+			"files"=>         ["json", "string"],
+			"images"=>        ["json", "string"],
+			"videos"=>        ["json", "string"],
+			"audios"=>        ["json", "string"],
+			"links"=>         ["json", "string"],
+			"domain"=>        ["empty", "string"],
+			"keywords"=>      ["json", "string"],
+			"lang"=>          ["empty", "string"],
+			"country_cn"=>    ["empty", "string"],
+			"country_code"=>  ["empty", "string"],
+			"created_at"=>    ["empty", "int", "length:13"],
+			"updated_at"=>    ["empty", "int", "length:13"],
+			"created_time"=>  ["empty", "int", "length:10"],
+			"oss_files"=>     ["json", "string"],
+			"oss_images"=>    ["json", "string"],
+			"topics"=>        ["json", "string"],
+			"tags"=>          ["string", "json"],
+			"authors"=>       ["json", "fields:author_id,author_name,author_url", "string"],
+			"timezone"=>      ["empty", 'regex:[\+|-]\d{4}', "string"],
+			"timezone_location"=> ["empty", "string"],	
+		}
+	end	
+
+end

data/lib/models/thinktank_information.rb

@@ -0,0 +1,163 @@
+class ThinktankInformation < BaseModel
+	attr_accessor :id
+	attr_accessor :title
+	attr_accessor :site_name
+	attr_accessor :site_name_cn
+	attr_accessor :abstract
+	attr_accessor :content
+	attr_accessor :author_names
+	attr_accessor :state_info
+	attr_accessor :source
+	attr_accessor :files
+	attr_accessor :images
+	attr_accessor :videos
+	attr_accessor :audios
+	attr_accessor :links
+	attr_accessor :domain
+	attr_accessor :keywords
+	attr_accessor :html_content
+	attr_accessor :lang
+	attr_accessor :country_cn
+	attr_accessor :country_code
+	attr_accessor :created_at
+	attr_accessor :updated_at
+	attr_accessor :created_time
+	attr_accessor :oss_files
+	attr_accessor :oss_images
+	attr_accessor :customer_category
+	attr_accessor :category
+	attr_accessor :topics
+	attr_accessor :tags
+	attr_accessor :views
+	attr_accessor :comments
+	attr_accessor :reference
+	attr_accessor :mention_country
+	attr_accessor :authors
+	attr_accessor :sub_title
+	attr_accessor :timezone
+	attr_accessor :timezone_location
+
+	def self.table_name
+		return "thinktank_informations"
+	end
+	register
+
+
+	def initialize(options={})
+		options = JSON.parse(options.to_json)
+		@id = options["id"]
+		@title = options["title"]
+		@site_name = options["site_name"]
+		@site_name_cn = options["site_name_cn"]
+		@abstract = options["abstract"]
+		@content = options["content"]
+		@author_names = options["author_names"]
+		@state_info = options["state_info"]
+		@source = options["source"]
+		@files = options["files"]
+		@images = options["images"]
+		@videos = options["videos"]
+		@audios = options["audios"]
+		@links = options["links"]
+		@domain = options["domain"]
+		@keywords = options["keywords"]
+		@html_content = options["html_content"]
+		@lang = options["lang"]
+		@country_cn = options["country_cn"]
+		@country_code = options["country_code"]
+		@created_at = options["created_at"]
+		@updated_at = options["updated_at"]
+		@created_time = options["created_time"]
+		@oss_files = options["oss_files"]
+		@oss_images = options["oss_images"]
+		@customer_category = options["customer_category"]
+		@category = options["category"]
+		@topics = options["topics"]
+		@tags = options["tags"]
+		@views = options["views"]
+		@comments = options["comments"]
+		@reference = options["reference"]
+		@mention_country = options["mention_country"]
+		@authors = options["authors"]
+		@sub_title = options["sub_title"]
+		@timezone = options["timezone"]
+		@timezone_location = options["timezone_location"]
+	end
+
+	def as_json
+		return {
+			id: @id,
+			title: @title,
+			site_name: @site_name,
+			site_name_cn: @site_name_cn,
+			abstract: @abstract,
+			content: @content,
+			author_names: @author_names,
+			state_info: @state_info,
+			source: @source,
+			files: @files,
+			images: @images,
+			videos: @videos,
+			links: @links,
+			domain: @domain,
+			keywords: @keywords,
+			html_content: @html_content,
+			lang: @lang,
+			country_cn: @country_cn,
+			country_code: @country_code,
+			created_at: @created_at,
+			updated_at: @updated_at,
+			created_time: @created_time,
+			oss_files: @oss_files,
+			oss_images: @oss_images,
+			customer_category: @customer_category,
+			category: @category,
+			topics: @topics,
+			tags: @tags,
+			views: @views,
+			comments: @comments,
+			reference: @reference,
+			mention_country: @mention_country,
+			authors: @authors,
+			sub_title: @sub_title,
+			timezone: @timezone,
+			timezone_location: @timezone_location
+
+		}
+	end
+
+	def to_json
+		return as_json.to_json
+	end
+
+	def self.verify_keys
+		return {
+			"id"=>            ["empty", "string"],
+			"title"=>         ["empty", "string"],
+			"site_name"=>     ["empty", "string"],
+			"site_name_cn"=>  ["empty", "string"],
+			"content"=>       ["empty", "string"],
+			"source"=>        ["empty", "string"],
+			"files"=>         ["json", "string"],
+			"images"=>        ["json", "string"],
+			"videos"=>        ["json", "string"],
+			"audios"=>        ["json", "string"],
+			"links"=>         ["json", "string"],
+			"domain"=>        ["empty", "string"],
+			"keywords"=>      ["json", "string"],
+			"lang"=>          ["empty", "string"],
+			"country_cn"=>    ["empty", "string"],
+			"country_code"=>  ["empty", "string"],
+			"created_at"=>    ["empty", "int", "length:13"],
+			"updated_at"=>    ["empty", "int", "length:13"],
+			"created_time"=>  ["empty", "int", "length:10"],
+			"oss_files"=>     ["json", "string"],
+			"oss_images"=>    ["json", "string"],
+			"topics"=>        ["json", "string"],
+			"tags"=>          ["string", "json"],
+			"authors"=>       ["json", "fields:author_id,author_name,author_url", "string"],
+			"timezone"=>      ["empty", 'regex:[\+|-]\d{4}', "string"],
+			"timezone_location"=> ["empty", "string"],	
+		}
+	end	
+end

data/lib/models/thinktank_report.rb

@@ -0,0 +1,161 @@
+class ThinktankReport < BaseModel
+	attr_accessor :id 
+	attr_accessor :title 
+	attr_accessor :site_name
+	attr_accessor :site_name_cn
+	attr_accessor :abstract
+	attr_accessor :content
+	attr_accessor :author_names
+	attr_accessor :source
+	attr_accessor :files
+	attr_accessor :images
+	attr_accessor :videos
+	attr_accessor :audios
+	attr_accessor :links
+	attr_accessor :domain
+	attr_accessor :keywords
+	attr_accessor :html_content
+	attr_accessor :lang
+	attr_accessor :country_cn
+	attr_accessor :country_code
+	attr_accessor :created_at
+	attr_accessor :updated_at
+	attr_accessor :created_time
+	attr_accessor :oss_files
+	attr_accessor :oss_images
+	attr_accessor :customer_category
+	attr_accessor :category
+	attr_accessor :topics
+	attr_accessor :tags
+	attr_accessor :views
+	attr_accessor :comments
+	attr_accessor :reference
+	attr_accessor :mention_country
+	attr_accessor :authors
+	attr_accessor :sub_title
+	attr_accessor :timezone
+	attr_accessor :timezone_location
+
+	def self.table_name
+		return "thinktank_reports"
+	end
+	register
+
+
+	def initialize(options = {})
+		options = JSON.parse(options.to_json)
+		@id = options["id"]
+		@title = options["title"]
+		@site_name = options["site_name"]
+		@site_name_cn = options["site_name_cn"]
+		@abstract = options["abstract"]
+		@content = options["content"]
+		@author_names = options["author_names"]
+		@source = options["source"]
+		@files = options["files"]
+		@images = options["images"]
+		@videos = options["videos"]
+		@audios = options["audios"]
+		@links = options["links"]
+		@domain = options["domain"]
+		@keywords = options["keywords"]
+		@html_content = options["html_content"]
+		@lang = options["lang"]
+		@country_cn = options["country_cn"]
+		@country_code = options["country_code"]
+		@created_at = options["created_at"]
+		@updated_at = options["updated_at"]
+		@created_time = options["created_time"]
+		@oss_files = options["oss_files"]
+		@oss_images = options["oss_images"]
+		@customer_category = options["customer_category"]
+		@category = options["category"]
+		@topics = options["topics"]
+		@tags = options["tags"]
+		@views = options["views"]
+		@comments = options["comments"]
+		@reference = options["reference"]
+		@mention_country = options["mention_country"]
+		@authors = options["authors"]
+		@sub_title = options["sub_title"]
+		@timezone = options["timezone"]
+		@timezone_location = options["timezone_location"]
+	end
+
+	def to_json
+		return as_json.to_json
+	end
+
+	def as_json
+		return {
+			id: @id,
+			title: @title,
+			site_name: @site_name,
+			site_name_cn: @site_name_cn,
+			abstract: @abstract,
+			content: @content,
+			author_names: @author_names,
+			source: @source,
+			files: @files,
+			images: @images,
+			videos: @videos,
+			audios: @audios,
+			links: @links,
+			domain: @domain,
+			keywords: @keywords,
+			html_content: @html_content,
+			lang: @lang,
+			country_cn: @country_cn,
+			country_code: @country_code,
+			created_at: @created_at,
+			updated_at: @updated_at,
+			created_time: @created_time,
+			oss_files: @oss_files,
+			oss_images: @oss_images,
+			customer_category: @customer_category,
+			category: @category,
+			topics: @topics,
+			tags: @tags,
+			views: @views,
+			comments: @comments,
+			reference: @reference,
+			mention_country: @mention_country,
+			authors: @authors,
+			sub_title: @sub_title,
+			timezone: @timezone,
+			timezone_location: @timezone_location
+		}
+	end
+
+	def self.verify_keys
+		return {
+			"id"=>            ["empty", "string"],
+			"title"=>         ["empty", "string"],
+			"site_name"=>     ["empty", "string"],
+			"site_name_cn"=>  ["empty", "string"],
+			"content"=>       ["empty", "string"],
+			"source"=>        ["empty", "string"],
+			"files"=>         ["json", "string"],
+			"images"=>        ["json", "string"],
+			"videos"=>        ["json", "string"],
+			"audios"=>        ["json", "string"],
+			"links"=>         ["json", "string"],
+			"domain"=>        ["empty", "string"],
+			"keywords"=>      ["json", "string"],
+			"lang"=>          ["empty", "string"],
+			"country_cn"=>    ["empty", "string"],
+			"country_code"=>  ["empty", "string"],
+			"created_at"=>    ["empty", "int", "length:13"],
+			"updated_at"=>    ["empty", "int", "length:13"],
+			"created_time"=>  ["empty", "int", "length:10"],
+			"oss_files"=>     ["json", "string"],
+			"oss_images"=>    ["json", "string"],
+			"topics"=>        ["json", "string"],
+			"tags"=>          ["string", "json"],
+			"authors"=>       ["json", "fields:author_id,author_name,author_url", "string"],
+			"timezone"=>      ["empty", 'regex:[\+|-]\d{4}', "string"],
+			"timezone_location"=> ["empty", "string"],	
+		}
+	end
+
+end

data/lib/verify/verify.rb

@@ -0,0 +1,93 @@
+class Verify
+	def self.IsVerified?(items=[],table_name="")
+
+		if table_name == "" && items.size > 0
+			table_name = switch_table(items[0])
+		elsif table_name != "" && items.size > 0 
+			if !match_fields?(items[0],table_name)
+				return
+			end
+		end
+
+		if table_name == ""
+			puts "未匹配，不验证"
+		else
+			puts "验证：#{table_name}"
+		end
+
+		if table_name != ""
+			kclass = Object.const_get $map_models[table_name]
+
+			if kclass == nil
+				puts "ERROR #{table_name} 验证 not Exist!"
+				return true
+			end
+
+			items.each do |item|
+				if !kclass.new(item).verify
+					return false
+				end
+			end			
+		end
+
+
+		return true
+	end
+
+	def self.switch_table(item={})
+		table_name = ""
+
+		item_keys = JSON.parse(item.to_json).keys
+		# puts "----#{item_keys}--"
+
+		$map_models.each do |k,v|
+			klass = Object.const_get v
+			klass_keys = klass.fields
+
+			# puts "----#{klass_keys}--"
+			sub_keys = item_keys - klass_keys
+			if sub_keys.size == 0 && item_keys.size == klass_keys.size
+				return k
+			end
+		end
+
+		return table_name
+	end
+
+	def self.match_fields?(item={},table_name)
+		if table_name == nil || table_name == ""
+			return false
+		end
+		item_keys = JSON.parse(item.to_json).keys
+		v = $map_models[table_name]
+
+		if v == nil 
+			puts "ERROR #{table_name} not Exist!"
+			return false
+		end
+
+		klass = Object.const_get v
+		klass_keys = klass.fields
+
+		# puts "----#{klass_keys}--"
+		sub_keys1 = item_keys - klass_keys
+		if sub_keys1.size > 0
+			puts "ERROR #{sub_keys1} do not belong #{table_name}"
+			return false
+		end
+
+		sub_keys2 = klass_keys - item_keys
+		if sub_keys2.size > 0
+			puts "ERROR #{sub_keys2} do not Exist!"
+			return false
+		end
+
+		return true
+
+	end
+
+	def self.table_names
+		return $map_models.keys
+	end
+
+end

data/ruby_sdk_test.rb

@@ -10,6 +10,11 @@ def main
 
 	CrawlabRubySdk.save_item(a)
 
+	# CrawlabRubySdk.save_item(a,"thinktank_experts")
+	# CrawlabRubySdk.save_item(a,"thinktank_expert_reports")
+	# CrawlabRubySdk.save_item(a,"thinktank_informations")
+	# CrawlabRubySdk.save_item(a,"thinktank_reports")
+
 	item = {
 		"id":                "dec9d5415409cc9275f5590c145c3ccf",
 		"title":             "Association of Selected State Policies and Requirements for Buprenorphine Treatment With Per Capita Months of Treatment",
@@ -50,7 +55,6 @@ def main
 	}
 	CrawlabRubySdk.save_item(item)
 
-
 end
 
-main
+# main

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: crawlab_ruby_sdk
 version: !ruby/object:Gem::Version
-  version: 0.1.6
+  version: 0.1.8
 platform: ruby
 authors:
 - min
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2023-06-17 00:00:00.000000000 Z
+date: 2023-07-11 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: grpc
@@ -90,12 +90,14 @@ files:
 - lib/entity/request_pb.rb
 - lib/entity/response_code_pb.rb
 - lib/entity/response_pb.rb
-- lib/entity/result_pb.rb
 - lib/entity/stream_message_code_pb.rb
-- lib/entity/stream_message_data_task_pb.rb
 - lib/entity/stream_message_pb.rb
-- lib/models/node_pb.rb
-- lib/models/task_pb.rb
+- lib/models/base.rb
+- lib/models/thinktank_expert.rb
+- lib/models/thinktank_expert_report.rb
+- lib/models/thinktank_information.rb
+- lib/models/thinktank_report.rb
+- lib/verify/verify.rb
 - ruby_sdk_test.rb
 homepage: https://github.com/rich-bro/crawlab_ruby_sdk
 licenses: []

data/lib/entity/result_pb.rb

@@ -1,15 +0,0 @@
-require 'google/protobuf'
-
-Google::Protobuf::DescriptorPool.generated_pool.build do
-  add_file("entity/result.proto", :syntax => :proto3) do
-    add_message "grpc.Result" do
-      optional :_tid, :string, 1
-      optional :name, :string, 2
-      optional :age, :string, 3
-    end
-  end
-end
-
-module Grpc
-  Result = ::Google::Protobuf::DescriptorPool.generated_pool.lookup("grpc.Result").msgclass
-end

data/lib/entity/stream_message_data_task_pb.rb

@@ -1,58 +0,0 @@
-# # Generated by the protocol buffer compiler.  DO NOT EDIT!
-# # source: entity/stream_message_data_task.proto
-
-# require 'google/protobuf'
-
-# Google::Protobuf::DescriptorPool.generated_pool.build do
-#   add_file(PWD + "/proto/entity/stream_message_data_task.proto", :syntax => :proto3) do
-#     add_message "grpc.StreamMessageDataTask" do
-#       optional :task_id, :string, 1
-#       optional :data, :string,:repeated, 2
-#     end
-#   end
-# end
-
-# module Grpc
-#   StreamMessageDataTask = ::Google::Protobuf::DescriptorPool.generated_pool.lookup("grpc.StreamMessageDataTask").msgclass
-# end
-
-
-# frozen_string_literal: true
-# Generated by the protocol buffer compiler.  DO NOT EDIT!
-# source: entity/stream_message_data_task.proto
-
-require 'google/protobuf'
-
-# require 'entity/result_pb'
-
-
-descriptor_data = "\n%entity/stream_message_data_task.proto\x12\x04grpc\x1a\x13\x65ntity/result.proto\"D\n\x15StreamMessageDataTask\x12\x0f\n\x07task_id\x18\x01 \x01(\t\x12\x1a\n\x04\x64\x61ta\x18\x02 \x03(\x0b\x32\x0c.grpc.ResultB\x08Z\x06.;grpcb\x06proto3"
-
-pool = Google::Protobuf::DescriptorPool.generated_pool
-
-begin
-  pool.add_serialized_file(descriptor_data)
-rescue TypeError => e
-  # Compatibility code: will be removed in the next major version.
-  require 'google/protobuf/descriptor_pb'
-  parsed = Google::Protobuf::FileDescriptorProto.decode(descriptor_data)
-  parsed.clear_dependency
-  serialized = parsed.class.encode(parsed)
-  file = pool.add_serialized_file(serialized)
-  warn "Warning: Protobuf detected an import path issue while loading generated file #{__FILE__}"
-  imports = [
-    ["grpc.Result", "entity/result.proto"],
-  ]
-  imports.each do |type_name, expected_filename|
-    import_file = pool.lookup(type_name).file_descriptor
-    if import_file.name != expected_filename
-      warn "- #{file.name} imports #{expected_filename}, but that import was loaded as #{import_file.name}"
-    end
-  end
-  warn "Each proto file must use a consistent fully-qualified name."
-  warn "This will become an error in the next major version."
-end
-
-module Grpc
-  StreamMessageDataTask = ::Google::Protobuf::DescriptorPool.generated_pool.lookup("grpc.StreamMessageDataTask").msgclass
-end

data/lib/models/node_pb.rb

@@ -1,27 +0,0 @@
-# Generated by the protocol buffer compiler.  DO NOT EDIT!
-# source: models/node.proto
-
-require 'google/protobuf'
-
-Google::Protobuf::DescriptorPool.generated_pool.build do
-  add_file("proto/models/node.proto", :syntax => :proto3) do
-    add_message "grpc.Node" do
-      optional :_id, :string, 1
-      optional :name, :string, 2
-      optional :ip, :string, 3
-      optional :port, :string, 5
-      optional :mac, :string, 6
-      optional :hostname, :string, 7
-      optional :description, :string, 8
-      optional :key, :string, 9
-      optional :is_master, :bool, 11
-      optional :update_ts, :string, 12
-      optional :create_ts, :string, 13
-      optional :update_ts_unix, :int64, 14
-    end
-  end
-end
-
-module Grpc
-  Node = ::Google::Protobuf::DescriptorPool.generated_pool.lookup("grpc.Node").msgclass
-end

data/lib/models/task_pb.rb

@@ -1,26 +0,0 @@
-# Generated by the protocol buffer compiler.  DO NOT EDIT!
-# source: models/task.proto
-
-require 'google/protobuf'
-
-Google::Protobuf::DescriptorPool.generated_pool.build do
-  add_file("models/task.proto", :syntax => :proto3) do
-    add_message "grpc.Task" do
-      optional :_id, :string, 1
-      optional :spider_id, :string, 2
-      optional :status, :string, 5
-      optional :node_id, :string, 6
-      optional :cmd, :string, 8
-      optional :param, :string, 9
-      optional :error, :string, 10
-      optional :pid, :int32, 16
-      optional :run_type, :string, 17
-      optional :schedule_id, :string, 18
-      optional :type, :string, 19
-    end
-  end
-end
-
-module Grpc
-  Task = ::Google::Protobuf::DescriptorPool.generated_pool.lookup("grpc.Task").msgclass
-end