instagram-crawler 0.2.1 → 0.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: e460402bed3e496955845e5920100d7b216edda7a94e5f0635a0f5770fc34a44
-  data.tar.gz: 2f51f10e8c9f4f1ff485232169b4b375f9c307b9b95b762fe24e4d8f048a875c
+  metadata.gz: d477137a184a3ac845344d26e54118733ea6d2f0e3bf3aacbd743b5f11db5e04
+  data.tar.gz: e1f145d7032addca16e3eba2654e35a7348a45e8e99c1a4c4d380d784cd83940
 SHA512:
-  metadata.gz: 91a28b2486726b71a6dd011845af1c7e2d0f09cbea26f50c86e8b59a285aded39606b98f083c3d43913c894fc0c97e465e6650780652ec5d8e32472cca98a970
-  data.tar.gz: 47b5d98b9d12b30fe466e86140ab96dbf3e3359616b7bfd73b9e60e3616145516f45b9e6b962c2f4c95700d2647bb08c481dd3cb3acc54e6d3e7e8b20e311506
+  metadata.gz: 65893d3b488960667d1ea31157a74e5785eae92550fcac8ae6cd28f6b6203bba12e87d04a9d0456ff4e2d86f8a3494f3dac05456d1a92c2111f44816828463df
+  data.tar.gz: 576b2455b75c6317ffbd9a6e5973e1c4ebbf362825946860d39fcc14f12176ab46724709814e61b625969f05aa50d6e4b0cc8c74962e6f76aaccc6f2c8f81c71

data/README.md

@@ -48,6 +48,14 @@ instagram-crawler -u <user_name>
 instagram-crawler -u <user_name> -d -a 20181120
 ```
 
+### Download files before this date (YYYYMMDD)
+
+`-b || --before `
+
+```ruby
+instagram-crawler -u <user_name> -d -b 20181120
+```
+
 ### Generate log file
 
 `-l || --log `
@@ -77,6 +85,7 @@ options:
     -u, --username USERNAME          Instagram username
     -d, --download                   Download files
     -a, --after DATE                 Download files after this date (YYYYMMDD)
+    -b, --before DATE                Download files before this date (YYYYMMDD)
     -l, --log                        Generate a log file in the current directory
     -P, --proxyname PROXYNAME        Specify proxyname of your proxy server
     -p, --port PORT                  Specify port of your proxy server (default port: 8080)
@@ -98,6 +107,15 @@ docker pull mgleon08/instagram-crawler
 docker run -it --rm -v $PWD/instagram-crawler:/instagram-crawler -e sessionid=$sessionid --name marvel mgleon08/instagram-crawler -u marvel -a 20181124 -d -l
 ```
 
+
+## Terms of Use
+
+[Instagram Terms of Use](https://www.instagram.com/about/legal/terms/before-january-19-2013/)
+
+> 9.You must not access Instagram's private API by any other means other than the Instagram application itself.  
+  10.You must not crawl, scrape, or otherwise cache any content from Instagram including but not limited to user profiles and photos.
+
+
 ## Contributing
 
 Bug reports and pull requests are welcome on GitHub at [`https://github.com/mgleon08/instagram-crawler/pulls`](https://github.com/mgleon08/instagram-crawler/pulls)

data/lib/instagram_crawler/config.rb

@@ -3,7 +3,7 @@ module InstagramCrawler
     @default_url = "https://www.instagram.com".freeze
     class << self
       attr_reader :default_url, :user_name, :base_url, :base_path,
-                  :log_path, :after_date, :parse_date
+                  :log_path, :after_date, :before_date, :parse_after_date, :parse_before_date
       attr_accessor :download, :proxyname
       attr_writer :port
 
@@ -16,7 +16,12 @@ module InstagramCrawler
 
       def after_date=(after_date)
         @after_date = after_date
-        @parse_date = Time.parse(after_date).to_i
+        @parse_after_date = Time.parse(after_date).to_i
+      end
+
+      def before_date=(before_date)
+        @before_date = before_date
+        @parse_before_date = Time.parse(before_date).to_i
       end
 
       def port

data/lib/instagram_crawler/parser/args.rb

@@ -19,6 +19,7 @@ module InstagramCrawler
         opts.on('-u', '--username USERNAME', 'Instagram username') { |user_name| Config.user_name = user_name }
         opts.on('-d', '--download', 'Download files') { |download| Config.download = true }
         opts.on('-a', '--after DATE', 'Download files after this date (YYYYMMDD)') { |after_date| Config.after_date = after_date }
+        opts.on('-b', '--before DATE', 'Download files before this date (YYYYMMDD)') { |before_date| Config.before_date = before_date }
         opts.on('-l', '--log', 'Generate a log file in the current directory') { self.log = true }
         opts.on('-P', '--proxyname PROXYNAME', 'Specify proxyname of your proxy server') { |proxyname| Config.proxyname = proxyname }
         opts.on('-p', '--port PORT', 'Specify port of your proxy server (default port: 8080)') { |port| Config.port = port }

data/lib/instagram_crawler/parser/base.rb

@@ -22,8 +22,8 @@ module InstagramCrawler
         Time.at(ts).strftime('%Y-%m-%dT%H:%M')
       end
 
-      def check_time(time)
-        if Config.after_date && (Config.parse_date > time)
+      def check_after_time(time)
+        if Config.after_date && (Config.parse_after_date > time)
           Logger.info "\nSuccess, the files after #{Config.after_date} have been downloaded!".light_green
           exit
         end

data/lib/instagram_crawler/parser/html.rb

@@ -45,7 +45,8 @@ module InstagramCrawler
       def loop_edges(edges)
         edges.each do |edge|
           node = edge["node"]
-          check_time(node["taken_at_timestamp"])
+          next if Config.before_date && (Config.parse_before_date < node["taken_at_timestamp"])
+          check_after_time(node["taken_at_timestamp"])
           time = parse_to_date(node["taken_at_timestamp"])
           page_url = "https://www.instagram.com/p/#{node["shortcode"]}/"
 

data/lib/instagram_crawler/parser/json.rb

@@ -26,7 +26,8 @@ module InstagramCrawler
       def loop_edges(edges)
         edges.each do |edge|
           node = edge["node"]
-          check_time(node["taken_at_timestamp"])
+          next if Config.before_date && (Config.parse_before_date < node["taken_at_timestamp"])
+          check_after_time(node["taken_at_timestamp"])
           time = parse_to_date(node["taken_at_timestamp"])
 
           if node["is_video"]

data/lib/instagram_crawler/version.rb

@@ -1,3 +1,3 @@
 module InstagramCrawler
-  VERSION = "0.2.1".freeze
+  VERSION = "0.3.0".freeze
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: instagram-crawler
 version: !ruby/object:Gem::Version
-  version: 0.2.1
+  version: 0.3.0
 platform: ruby
 authors:
 - Leon Ji
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2019-03-27 00:00:00.000000000 Z
+date: 2019-04-14 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler