fofa 0.3.12 → 0.3.19

checksums.yaml

@@ -1,7 +1,7 @@
 ---
-SHA1:
-  metadata.gz: 87d0a961aecc981843e098178b7c3fc2c70c14f7
-  data.tar.gz: 10df9f621ee857e19eb9a31ea5c488fe71895135
+SHA256:
+  metadata.gz: bfc764bb2b5c3d56f6dfd68ac14a74478a7d19c4e14c72bec35b1d3f547fb7aa
+  data.tar.gz: 8c52a5488290a45f3c074fcdccca99bef75685e91d42fe554268eb886cad36f3
 SHA512:
-  metadata.gz: 4c2eaf481b20f662ecac532f950c866ca7fc11d966e4a8b067ad67faca035e08d305923c6ad3a5c85025f3da7d3afc247b91461d4cbb2590cc126f26f28034b2
-  data.tar.gz: f639fd08740c4df9bcece0443a4b802d9d8df01de17c5136abe4c9fdf4b1a5b6bb09e7c6f04c3c4bd9ddd6127df39ff5aa67e98de72d550f7cf4aefc81193028
+  metadata.gz: 67ba297c2bba208e41d01700549cb396de980fb3937e932e01699ee90ce0185d2e9f6daac48fd9366ac919adbc883cd3377686ec5b5fff15e5b44c2ff594e381
+  data.tar.gz: c998095f14e85ceeb262b4f6b3b02c102d3a75dd0d2c0088c6b0402b7e8b968e0311a950261c91c45719d80fb02d3e5910da3106d1c8470dd1ab04eba3eaf03a

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    fofa (0.3.11)
+    fofa (0.3.19)
       colorize
       concurrent-ruby
 
@@ -38,4 +38,4 @@ DEPENDENCIES
   rspec (~> 3.0)
 
 BUNDLED WITH
-   1.14.6
+   1.16.4

data/bin/fofacli

@@ -18,7 +18,11 @@ options = {
   fields: 'host',
   post: false,
   query: nil,
-	format: 'csv'
+	format: 'csv',
+	check_app_category: nil,
+  check_app_application: nil,
+	check_app_all: false,
+	task_id: nil
 }
 
 ARGV.options do |opts|
@@ -39,12 +43,16 @@ ARGV.options do |opts|
 			options[:page] = val.to_i
 	end
 
-	opts.on('-m', '--mode=MODE', String, 'Mode, default to [search], -m should be specified when [import_service, query_ip_list] ') do |val|
+	opts.on('-m', '--mode=MODE', String, 'Mode, default to [search], -m should be specified when [import_service, query_ip_list, check_app, ip_tags] ') do |val|
 			options[:mode] = val.to_sym
 	end
 
-	opts.on('-f', '--file=FILE', String, 'Used at [import_service] mode') do |val|
+	opts.on('-f', '--file=FILE', String, 'Used at [import_service, check_app, ip_tags] mode') do |val|
 			options[:file] = val
+  end
+
+	opts.on('-t', '--task_id=ID', Integer, 'Used at [ip_tags] mode') do |val|
+		options[:task_id] = val.to_i
 	end
 
 	opts.on('-s', '--limit=SIZE', Integer, "Limit size to fetch, default to #{options[:limit]}") do |val|
@@ -67,7 +75,17 @@ ARGV.options do |opts|
 		raise "File not exist of #{val}" unless File.exists? val
     options[:query] = File.read(val).strip
 		options[:post] = true
-	end
+  end
+
+	opts.on('-l', "--application=APPLICATION", String, "Application to check, only used in check_app mode." ) do |val|
+		options[:check_app_application] = val
+		options[:mode] = :check_app
+  end
+
+	opts.on('-g', "--category=CATEGORY", String, "Category to check, only used in check_app mode." ) do |val|
+		options[:check_app_category] = val
+		options[:mode] = :check_app
+  end
 	
 	opts.separator "Common Options:"
 
@@ -84,13 +102,23 @@ ARGV.options do |opts|
 		options[:verbose] = true
   end
 
+	opts.on( nil, "--check_app_all", "Check all applications, only used in check_app mode." ) do
+		options[:check_app_all] = true
+  end
+
+
+
 	opts.separator ""
 	opts.separator "For example:"
 	opts.separator "\t fofacli -e aaa@bbb.com -a xxx domain=\"baidu.com\""
 	opts.separator "\t fofacli -e aaa@bbb.com -a xxx -s 100 domain=\"baidu.com\""
 	opts.separator "\t fofacli -e aaa@bbb.com -a xxx --query_file /tmp/fofaquery"
 	opts.separator "\t fofacli -e aaa@bbb.com -a xxx -d ip,domain,title,port,protocol -m query_ip_list -f ip.txt"
-	
+	opts.separator %Q@\t cat vapps.txt | xargs -I{} -n 1 sh -c 'fofacli "app=$1" -d host,ip,port,country,province,city -s 1000 -o json -v > "$1.txt"' -- {}@
+	opts.separator "\t fofacli -m check_app --app \"Coremail\" mail.tsinghua.edu.cn"
+	opts.separator "\t fofacli -e aaa@bbb.com -a xxx -m ip_tags -f ./ip.txt"
+	opts.separator "\t fofacli -e aaa@bbb.com -a xxx -m ip_tags -t 1"
+
 	begin
 		opts.parse!
 	rescue OptionParser::InvalidOption => e
@@ -125,8 +153,13 @@ def log_record(options, r)
 			else
 				puts %Q|{"id":"#{r}"}|
 			end
-		when 'csv'
-			puts r.map{|v| v}.join("\t")
+    when 'csv'
+      if r.kind_of?(Array)
+			  puts r.map{|v| v}.join("\t")
+      else
+        puts r
+      end
+
 	end
 end
 
@@ -221,5 +254,86 @@ case options[:mode]
     } #File
 		pool.shutdown
 		pool.wait_for_termination
+  when :check_app
+		query = options[:query] || ARGV.join(' ')
+		fofa = Fofa::API.new(options[:email], options[:apikey], {debug:options[:verbose]})
+		if options[:file]
+			File.open(options[:file]){|f|
+        f.each_line{|line|
+					query = line.strip
+					res = fofa.check_app(query, {
+							category:options[:check_app_category],
+							application:options[:check_app_application],
+							all:options[:all],
+					})
+
+          if res && res.size>0
+						res.each{|r|
+							log_record(options, [query, r])
+						}
+          else
+						log_record(options, [query, nil])
+          end
+
+        }
+      }
+    else
+			res = fofa.check_app(query, {
+					category:options[:check_app_category],
+					application:options[:check_app_application],
+					all:options[:all],
+			})
+
+			if res && res.size>0
+				res.each{|r|
+					log_record(options, r)
+				}
+      else
+        $stderr.puts "[WARNING] No result!"
+      end
+
+    end
+	when :ip_tags
+		fofa = Fofa::API.new(options[:email], options[:apikey], {debug:options[:verbose]})
+    if options[:task_id]
+			task_id = options[:task_id]
+    else
+			unless options[:file]
+				puts "File not specified.".red
+				puts ARGV.options
+				exit -1
+			end
+			puts "Import ip and get tags from '#{options[:file]}'"
+			result = fofa.ip_tags(options[:file])
+			if result['error']
+				puts result
+				exit -1
+      end
+			puts result
+			task_id = result['task_id']
+    end
+
+    if task_id
+      get_info = -> (fofa, id) { fofa.tag_info(id) }
+      puts "Checking progress!"
+      sleep(2)
+
+      loop do
+				info = get_info.call(fofa, task_id)
+				if info['error']
+					puts info
+					break
+        elsif info['progress'].to_i == 100
+					puts "state: #{info['state']}; progress: #{info['progress']}."
+					puts info['download_url']
+					break
+        else
+          puts "state: #{info['state']}; progress: #{info['progress']}."
+          sleep(10)
+				end
+      end
+
+    end
+
 end
 

data/lib/fofa.rb

@@ -1,6 +1,7 @@
 require "fofa/version"
 require 'net/http'
 require 'json'
+require 'base64'
 
 module Fofa
   class API
@@ -29,13 +30,10 @@ module Fofa
         options = {page:1, size:100, fields:'host'}.merge(options)
 
     		url = "#{@api_server}/api/v1/search/all?key=#{@apikey}&email=#{@email}&page=#{options[:page]}&size=#{options[:size]}&fields=#{options[:fields]}"
-    		url += "&q=#{URI.escape(query)}" unless options[:post]
+    		url += "&qbase64=#{Base64.encode64(query)}" unless options[:post]
         puts url if @options[:debug]
     		uri = URI.parse(url)
-        http = Net::HTTP.new(uri.host, uri.port)
-        if uri.scheme == 'https'
-          http.use_ssl = true
-        end
+        http = http_new(uri)
 
         if options[:post]
           req = Net::HTTP::Post.new(uri.request_uri)
@@ -47,6 +45,7 @@ module Fofa
         resp = http.request(req)
         JSON.parse(resp.body)
     rescue => e
+      $stderr.puts "[WARNING]: #{e.to_s}"
     	{"error"=>"Error: #{e.to_s}"}
     end
 
@@ -74,7 +73,7 @@ module Fofa
         res = search(query, {page:page}.merge(options))
         if !res['error'] && res['results'].size > 0
           all_res += res['results']
-          yield(res['results'], page) if block_given?
+          yield(res['results'], page, res['size'].to_i) if block_given?
 
           if all_size && all_res.size > all_size.to_i
             all_res = all_res[0..all_size-1]
@@ -104,11 +103,7 @@ module Fofa
       url = "#{@api_server}/api/v1/import/services?key=#{@apikey}&email=#{@email}&port=#{options[:port]}"
       puts url if @options[:debug]
       uri = URI.parse(url)
-      http = Net::HTTP.new(uri.host, uri.port)
-      if uri.scheme == 'https'
-        http.use_ssl = true
-      end
-      http.set_debug_output $stderr if @options[:debug]
+      http = http_new(uri)
 
       File.open(file) do |f|
         results = [] 
@@ -137,5 +132,99 @@ module Fofa
       {"error"=>"Error: #{e.to_s}"}
     end
 
+    # Check applications of asset
+    #
+    # Example:
+    #   >> Fofa::API.new(email,apikey).checkapp("mail.tsinghua.edu.cn", category:"邮件系统")
+    #   => ["Coremail"]
+    #
+    # Arguments:
+    #   host: (String) Category name
+    #   options: (Hash) category: Category name, application: Application name, all: return all applications or break when match first
+    def check_app(host, options={})
+      options = {all:false}.merge(options)
+      url = "#{@api_server}/api/v1/search/checkapp?key=#{@apikey}&email=#{@email}&host=#{host}"
+      url += "&all=#{options[:all]}" if options[:all]
+      url += "&application=#{URI.escape(options[:application])}" if options[:application]
+      url += "&category=#{URI.escape(options[:category])}" if options[:category]
+      puts url if @options[:debug]
+      uri = URI.parse(url)
+      http = http_new(uri)
+      req = Net::HTTP::Get.new(uri.request_uri)
+      resp = http.request(req)
+      JSON.parse(resp.body)
+    rescue => e
+      {"error"=>"Error: #{e.to_s}"}
+    end
+
+    # Import ip and get tags
+    #
+    # Example:
+    #   >> Fofa::API.new(email,apikey).ip_tags('./ips.txt')
+    #   => {"task_id"=>1}
+    #
+    # Arguments:
+    #   file: (String) ip file
+    def ip_tags(file)
+      url = "#{@api_server}/api/v1/ip_tags?key=#{@apikey}&email=#{@email}"
+      puts url if @options[:debug]
+
+      # Token used to terminate the file in the post body. Make sure it is not
+      # present in the file you're uploading.
+      # You might want to use `SecureRandom` class to generate this random strings
+      boundary = "AaB03x"
+      uri = URI.parse(url)
+
+      post_body = []
+      post_body << "--#{boundary}\r\n"
+      post_body << "Content-Disposition: form-data; name=\"file\"; filename=\"#{File.basename(file)}\"\r\n"
+      post_body << "Content-Type: text/plain\r\n"
+      post_body << "\r\n"
+      post_body << File.read(file)
+      post_body << "\r\n--#{boundary}--\r\n"
+
+      http = http_new(uri)
+      req = Net::HTTP::Post.new(uri.request_uri)
+      req.body = post_body.join
+      req["Content-Type"] = "multipart/form-data, boundary=#{boundary}"
+
+      resp = http.request(req)
+      puts resp if @options[:debug]
+      JSON.parse(resp.body)
+    rescue => e
+      {"error"=>"Error: #{e.to_s}"}
+    end
+
+    # Check ip_tags's task state, progress, and download_url
+    #
+    # Example:
+    #   >> Fofa::API.new(email,apikey).tag_info(1)
+    #   => {"state"=>"success", "progress"=>100, "download_url"=>"https://host/xxx/56af409c1c55762a7b9e9a39a801f80d.json"}
+    #
+    # Arguments:
+    #   task_id: (Integer) task id
+    def tag_info(task_id)
+      url = "#{@api_server}/api/v1/ip_tags/info?key=#{@apikey}&email=#{@email}&task_id=#{task_id}"
+      puts url if @options[:debug]
+      uri = URI.parse(url)
+      http = http_new(uri)
+      req = Net::HTTP::Get.new(uri.request_uri)
+
+      resp = http.request(req)
+      JSON.parse(resp.body)
+    rescue => e
+      $stderr.puts "[WARNING]: #{e.to_s}"
+      {"error"=>"Error: #{e.to_s}"}
+    end
+
+    private
+    def http_new(uri)
+      http = Net::HTTP.new(uri.host, uri.port)
+      if uri.scheme == 'https'
+        http.use_ssl = true
+      end
+      http.set_debug_output $stderr if @options[:debug]
+      http
+    end
   end
 end

data/lib/fofa/version.rb

@@ -1,3 +1,21 @@
+# 0.3.19
+#   add feature: ip_tags, tag_info
+#
+# 0.3.18
+#   add feature: check applications from file
+#
+# 0.3.17
+#   add checkapp api call
+#
+# 0.3.15
+#   fixed query encode bug of '&&'
+#
+# 0.3.14
+#   fixed bug of search_all yield when size reached
+#
+# 0.3.13
+#   search_all yield with all size
+#
 # 0.3.12
 #   fofacli add -o --output-format option, support csv(defualt) and json
 #
@@ -16,5 +34,5 @@
 # 0.3.6
 #   add fields
 module Fofa
-  VERSION = "0.3.12"
+  VERSION = "0.3.19"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fofa
 version: !ruby/object:Gem::Version
-  version: 0.3.12
+  version: 0.3.19
 platform: ruby
 authors:
 - fofa
-autorequire: 
+autorequire:
 bindir: bin
 cert_chain: []
-date: 2017-05-11 00:00:00.000000000 Z
+date: 2020-06-29 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -106,7 +106,7 @@ homepage: https://fofa.so
 licenses:
 - MIT
 metadata: {}
-post_install_message: 
+post_install_message:
 rdoc_options: []
 require_paths:
 - lib
@@ -121,9 +121,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubyforge_project: 
-rubygems_version: 2.6.3
-signing_key: 
+rubygems_version: 3.0.6
+signing_key:
 specification_version: 4
 summary: A Ruby library to interact with the FOFA API. https://fofa.so
 test_files: []