fofa 0.3.18 → 0.4.22

checksums.yaml

@@ -1,7 +1,7 @@
 ---
-SHA1:
-  metadata.gz: d1266f6e972edc71cf6f3c36c8dca778832d9340
-  data.tar.gz: 9e4c2976d711e4e43dfcd18878e6518434d5a7be
+SHA256:
+  metadata.gz: e19e36559d8749989cb0c93e56bbcd33ce8c28c5cf1da6923e48206c12b897c3
+  data.tar.gz: a94b322821b82a56ac996701aa199dfd3ebbeb90165fd57226244ac0148be626
 SHA512:
-  metadata.gz: 077e078e08a70d059fed83f8f92cd425096172c160ab0eab71702eb9026a1328bb812a7b7cb7cbf03f1eb9e239a94258b138513bcb275f26351c088b32d8b36e
-  data.tar.gz: 892e27388b7cb3a1be9de0ec36d8b20c5c0223b1d481432218a61a6beeb86ebfcf9ddd6d464e4fb4e47fcc5994bed1ce9467981031d030ea23ce0874ec408fa9
+  metadata.gz: 1de683c6f0e5a8c9752c4a14ad70dee53da5824ded7e36f32155f2b57e381789038f51a6d661225dfb230573413b03b815a7fc441acfb5d445a0f6489ddb18cc
+  data.tar.gz: 5e23cc12d117ea88456fe1ea15a1f6c177ef780370ffd8420c859af10007fa544c36b7fb07ef26b3fb914082f108e5795d4be00eef98f362982a29df6f2d1e60

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    fofa (0.3.17)
+    fofa (0.4.21)
       colorize
       concurrent-ruby
 
@@ -38,4 +38,4 @@ DEPENDENCIES
   rspec (~> 3.0)
 
 BUNDLED WITH
-   1.15.1
+   1.16.4

data/bin/fofacli

@@ -19,9 +19,7 @@ options = {
   post: false,
   query: nil,
 	format: 'csv',
-	check_app_category: nil,
-  check_app_application: nil,
-	check_app_all: false
+	task_id: nil
 }
 
 ARGV.options do |opts|
@@ -42,12 +40,16 @@ ARGV.options do |opts|
 			options[:page] = val.to_i
 	end
 
-	opts.on('-m', '--mode=MODE', String, 'Mode, default to [search], -m should be specified when [import_service, query_ip_list, check_app] ') do |val|
+	opts.on('-m', '--mode=MODE', String, 'Mode, default to [search], -m should be specified when [import_service, query_ip_list, stats, ip_tags] ') do |val|
 			options[:mode] = val.to_sym
 	end
 
-	opts.on('-f', '--file=FILE', String, 'Used at [import_service, check_app] mode') do |val|
+	opts.on('-f', '--file=FILE', String, 'Used at [import_service, ip_tags] mode') do |val|
 			options[:file] = val
+  end
+
+	opts.on('-t', '--task_id=ID', Integer, 'Used at [ip_tags] mode') do |val|
+		options[:task_id] = val.to_i
 	end
 
 	opts.on('-s', '--limit=SIZE', Integer, "Limit size to fetch, default to #{options[:limit]}") do |val|
@@ -72,16 +74,6 @@ ARGV.options do |opts|
 		options[:post] = true
   end
 
-	opts.on('-l', "--application=APPLICATION", String, "Application to check, only used in check_app mode." ) do |val|
-		options[:check_app_application] = val
-		options[:mode] = :check_app
-  end
-
-	opts.on('-g', "--category=CATEGORY", String, "Category to check, only used in check_app mode." ) do |val|
-		options[:check_app_category] = val
-		options[:mode] = :check_app
-	end
-	
 	opts.separator "Common Options:"
 
 	opts.on( nil, "--post", "POST query mode." ) do
@@ -97,12 +89,11 @@ ARGV.options do |opts|
 		options[:verbose] = true
   end
 
-	opts.on( nil, "--check_app_all", "Check all applications, only used in check_app mode." ) do
-		options[:check_app_all] = true
+	opts.on( "-V", "--version", "Display version" ) do
+    puts "Version: #{Fofa::VERSION}"
+    exit(0)
 	end
 
-
-
 	opts.separator ""
 	opts.separator "For example:"
 	opts.separator "\t fofacli -e aaa@bbb.com -a xxx domain=\"baidu.com\""
@@ -110,7 +101,9 @@ ARGV.options do |opts|
 	opts.separator "\t fofacli -e aaa@bbb.com -a xxx --query_file /tmp/fofaquery"
 	opts.separator "\t fofacli -e aaa@bbb.com -a xxx -d ip,domain,title,port,protocol -m query_ip_list -f ip.txt"
 	opts.separator %Q@\t cat vapps.txt | xargs -I{} -n 1 sh -c 'fofacli "app=$1" -d host,ip,port,country,province,city -s 1000 -o json -v > "$1.txt"' -- {}@
-	opts.separator "\t fofacli -m check_app --app \"Coremail\" mail.tsinghua.edu.cn"
+	opts.separator "\t fofacli -m stats --fields domain app=\"drupal\" | jq"
+	opts.separator "\t fofacli -e aaa@bbb.com -a xxx -m ip_tags -f ./ip.txt"
+	opts.separator "\t fofacli -e aaa@bbb.com -a xxx -m ip_tags -t 1"
 
 	begin
 		opts.parse!
@@ -140,17 +133,17 @@ def log_record(options, r)
 	case options[:format]
 		when 'json'
 			if r.kind_of?(Array)
-				puts Hash[options[:fields].split(',').zip r].to_json
+				STDOUT.puts Hash[options[:fields].split(',').zip r].to_json
 			elsif r.kind_of?(Hash)
-				puts r.to_json
+				STDOUT.puts r.to_json
 			else
-				puts %Q|{"id":"#{r}"}|
+				STDOUT.puts %Q|{"id":"#{r}"}|
 			end
     when 'csv'
       if r.kind_of?(Array)
-			  puts r.map{|v| v}.join("\t")
+				STDOUT.puts r.map{|v| v}.join("\t")
       else
-        puts r
+				STDOUT.puts r
       end
 
 	end
@@ -164,7 +157,7 @@ case options[:mode]
 			exit -1
 		end
 		query = options[:query] || ARGV.join(' ')
-		puts "Query: '#{query}'"
+		STDERR.puts "Query: '#{query}'"
 		if options[:page] #search one page
 			Fofa::API.new(options[:email], options[:apikey], {debug:options[:verbose]})
 							 .search(query, {
@@ -206,7 +199,7 @@ case options[:mode]
 		require 'concurrent'
 		semaphore = Concurrent::Semaphore.new(1)
 		pool = Concurrent::FixedThreadPool.new(10)
-    $stdout.sync = true
+    $STDOUT.sync = true
     File.open(options[:file]){|f|
       fields = options[:fields].split(',')
       f.each_line{|line|
@@ -247,46 +240,60 @@ case options[:mode]
     } #File
 		pool.shutdown
 		pool.wait_for_termination
-  when :check_app
+  when :stats
 		query = options[:query] || ARGV.join(' ')
 		fofa = Fofa::API.new(options[:email], options[:apikey], {debug:options[:verbose]})
-		if options[:file]
-			File.open(options[:file]){|f|
-        f.each_line{|line|
-					query = line.strip
-					res = fofa.check_app(query, {
-							category:options[:check_app_category],
-							application:options[:check_app_application],
-							all:options[:all],
-					})
-
-          if res && res.size>0
-						res.each{|r|
-							log_record(options, [query, r])
-						}
-          else
-						log_record(options, [query, nil])
-          end
+		res = fofa.stats(query, {
+      fields: options[:fields],
+		})
 
-        }
-      }
+		if res && res.size>0
+      $stdout.puts Hash[res].to_json
+		else
+			$stderr.puts "[WARNING] No result!"
+		end
+
+	when :ip_tags
+		fofa = Fofa::API.new(options[:email], options[:apikey], {debug:options[:verbose]})
+    if options[:task_id]
+			task_id = options[:task_id]
     else
-			res = fofa.check_app(query, {
-					category:options[:check_app_category],
-					application:options[:check_app_application],
-					all:options[:all],
-			})
-
-			if res && res.size>0
-				res.each{|r|
-					log_record(options, r)
-				}
-      else
-        $stderr.puts "[WARNING] No result!"
+			unless options[:file]
+				puts "File not specified.".red
+				puts ARGV.options
+				exit -1
+			end
+			puts "Import ip and get tags from '#{options[:file]}'"
+			result = fofa.ip_tags(options[:file])
+			if result['error']
+				puts result
+				exit -1
       end
-
+			puts result
+			task_id = result['task_id']
     end
 
+    if task_id
+      get_info = -> (fofa, id) { fofa.tag_info(id) }
+      puts "Checking progress!"
+      sleep(2)
+
+      loop do
+				info = get_info.call(fofa, task_id)
+				if info['error']
+					puts info
+					break
+        elsif info['progress'].to_i == 100
+					puts "state: #{info['state']}; progress: #{info['progress']}."
+					puts info['download_url']
+					break
+        else
+          puts "state: #{info['state']}; progress: #{info['progress']}."
+          sleep(10)
+				end
+      end
+
+    end
 
 end
 

data/lib/fofa/version.rb

@@ -1,3 +1,15 @@
+# 0.4.22
+#   add state api, remove chech_app api
+#
+# 0.4.21
+#   split result to stdout
+#
+# 0.4.20
+#   add -V option, and ip_tags method need to upgrade major version
+#
+# 0.3.19
+#   add feature: ip_tags, tag_info
+#
 # 0.3.18
 #   add feature: check applications from file
 #
@@ -31,5 +43,5 @@
 # 0.3.6
 #   add fields
 module Fofa
-  VERSION = "0.3.18"
+  VERSION = "0.4.22"
 end

data/lib/fofa.rb

@@ -132,28 +132,86 @@ module Fofa
       {"error"=>"Error: #{e.to_s}"}
     end
 
-    # Check applications of asset
+    # Fetch fids from query
     #
     # Example:
-    #   >> Fofa::API.new(email,apikey).checkapp("mail.tsinghua.edu.cn", category:"邮件系统")
+    #   >> Fofa::API.new(email,apikey).stats("domain=baidu.com", fields:"fid")
     #   => ["Coremail"]
     #
     # Arguments:
-    #   host: (String) Category name
-    #   options: (Hash) category: Category name, application: Application name, all: return all applications or break when match first
-    def check_app(host, options={})
-      options = {all:false}.merge(options)
-      url = "#{@api_server}/api/v1/search/checkapp?key=#{@apikey}&email=#{@email}&host=#{host}"
-      url += "&all=#{options[:all]}" if options[:all]
-      url += "&application=#{URI.escape(options[:application])}" if options[:application]
-      url += "&category=#{URI.escape(options[:category])}" if options[:category]
+    #   query: (String) fofa query string
+    #   options: (Hash) fields: can be return multiple fields, like fid,host and so on
+    def stats(query, options={})
+      options = {fields:"fid"}.merge(options)
+      url = "#{@api_server}/api/v1/search/stats?key=#{@apikey}&email=#{@email}&qbase64=#{Base64.encode64(query)}"
+      url += "&fields=#{options[:fields]}" if options[:fields]
+      puts url if @options[:debug]
+      uri = URI.parse(url)
+      http = http_new(uri)
+      req = Net::HTTP::Get.new(uri.request_uri)
+      resp = http.request(req)
+      JSON.parse(resp.body)
+    rescue => e
+      {"error"=>"Error: #{e.to_s}"}
+    end
+
+    # Import ip and get tags
+    #
+    # Example:
+    #   >> Fofa::API.new(email,apikey).ip_tags('./ips.txt')
+    #   => {"task_id"=>1}
+    #
+    # Arguments:
+    #   file: (String) ip file
+    def ip_tags(file)
+      url = "#{@api_server}/api/v1/ip_tags?key=#{@apikey}&email=#{@email}"
+      puts url if @options[:debug]
+
+      # Token used to terminate the file in the post body. Make sure it is not
+      # present in the file you're uploading.
+      # You might want to use `SecureRandom` class to generate this random strings
+      boundary = "AaB03x"
+      uri = URI.parse(url)
+
+      post_body = []
+      post_body << "--#{boundary}\r\n"
+      post_body << "Content-Disposition: form-data; name=\"file\"; filename=\"#{File.basename(file)}\"\r\n"
+      post_body << "Content-Type: text/plain\r\n"
+      post_body << "\r\n"
+      post_body << File.read(file)
+      post_body << "\r\n--#{boundary}--\r\n"
+
+      http = http_new(uri)
+      req = Net::HTTP::Post.new(uri.request_uri)
+      req.body = post_body.join
+      req["Content-Type"] = "multipart/form-data, boundary=#{boundary}"
+
+      resp = http.request(req)
+      puts resp if @options[:debug]
+      JSON.parse(resp.body)
+    rescue => e
+      {"error"=>"Error: #{e.to_s}"}
+    end
+
+    # Check ip_tags's task state, progress, and download_url
+    #
+    # Example:
+    #   >> Fofa::API.new(email,apikey).tag_info(1)
+    #   => {"state"=>"success", "progress"=>100, "download_url"=>"https://host/xxx/56af409c1c55762a7b9e9a39a801f80d.json"}
+    #
+    # Arguments:
+    #   task_id: (Integer) task id
+    def tag_info(task_id)
+      url = "#{@api_server}/api/v1/ip_tags/info?key=#{@apikey}&email=#{@email}&task_id=#{task_id}"
       puts url if @options[:debug]
       uri = URI.parse(url)
       http = http_new(uri)
       req = Net::HTTP::Get.new(uri.request_uri)
+
       resp = http.request(req)
       JSON.parse(resp.body)
     rescue => e
+      $stderr.puts "[WARNING]: #{e.to_s}"
       {"error"=>"Error: #{e.to_s}"}
     end
 

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fofa
 version: !ruby/object:Gem::Version
-  version: 0.3.18
+  version: 0.4.22
 platform: ruby
 authors:
 - fofa
-autorequire: 
+autorequire:
 bindir: bin
 cert_chain: []
-date: 2017-09-09 00:00:00.000000000 Z
+date: 2022-02-15 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -106,7 +106,7 @@ homepage: https://fofa.so
 licenses:
 - MIT
 metadata: {}
-post_install_message: 
+post_install_message:
 rdoc_options: []
 require_paths:
 - lib
@@ -121,9 +121,9 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubyforge_project: 
-rubygems_version: 2.6.12
-signing_key: 
+rubyforge_project:
+rubygems_version: 2.7.7
+signing_key:
 specification_version: 4
 summary: A Ruby library to interact with the FOFA API. https://fofa.so
 test_files: []