social_miner 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: dceb80b76ae8ca6044e919608765d26e0e76508672b9a2bfe5e3179d05a0a040
+  data.tar.gz: 32da50a190e4ac98d30f88d5710ad2883a3bcf3eedc1a66d8e3e2bb96d20bcfa
+SHA512:
+  metadata.gz: 931ed6f7546d1a477489b25d378c6b7c444ed6087c1323b2ad4c8fb27d47aca6d03d8ec0a287fc37e529c991c56c579c05c43b1efb7ee23cc0270f2c6c6bea84
+  data.tar.gz: 7c063258a95a5e6c0f3339652d1823a5521f3b24f6e959eba5afaaf6e88d5e14f678dcf5af5d110db857b91bde991f13fa96964617828624c0037422f777cd5a

data/.rspec

@@ -0,0 +1,3 @@
+--format documentation
+--color
+--require spec_helper

data/.rubocop.yml

@@ -0,0 +1,21 @@
+AllCops:
+  NewCops: enable
+  TargetRubyVersion: 3.2
+
+Style/StringLiterals:
+  EnforcedStyle: double_quotes
+
+Style/StringLiteralsInInterpolation:
+  EnforcedStyle: double_quotes
+
+Style/Documentation:
+  Enabled: false
+  
+Metrics/BlockLength:
+  Max: 70
+
+Metrics/MethodLength:
+  Max: 30
+
+Metrics/AbcSize:
+  Max: 30

data/CHANGELOG.md

@@ -0,0 +1,3 @@
+## [0.1.0] - 2024-10-31
+
+- Initial release

data/LICENSE.txt

@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright (c) 2024 Null0vector
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

data/README.md

@@ -0,0 +1,216 @@
+# SocialMiner
+
+SocialMiner is a Ruby gem that provides a simple interface to fetch public data from Instagram, including user profiles, posts, and comments.
+
+# Installation
+
+Add this line to your application's Gemfile:
+
+```ruby
+gem 'social_miner'
+```
+
+And then execute:
+
+```bash
+bundle install
+```
+
+# Usage
+Fetch user profile info:
+
+```ruby
+mapper = SocialMiner::Instagram::ProfileMapper
+
+result = SocialMiner::Instagram.profile_info(username: "instagram") do |profile_attrs|
+  mapper.map(profile_attrs)
+end
+# Result contains:
+# {
+#   social_id: "123456789",
+#   username: "instagram_username",
+#   description: "User bio",
+#   first_name: "John",
+#   last_name: "Doe",
+#   avatar_url: "https://...",
+#   followers: 1000,
+#   following: 500
+# }
+```
+
+To fetch a user's posts:
+
+```ruby
+mapper = SocialMiner::Instagram::PostMapper
+# User ID can be obtained from the profile info (social_id field)
+result, cursor = SocialMiner::Instagram.profile_posts(user_id: "user_id") do |posts_attrs, cursor|
+  [
+    posts_attrs.map { |post_attrs| mapper.map(post_attrs) },
+    cursor
+  ]
+end
+
+# Result contains:
+# {
+#   cursor: "next_page_token",
+#   records: [
+#     {
+#       social_id: "post123",
+#       image_url: "https://...",
+#       shortcode: "ABC123",
+#       location_name: "New York",
+#       description: "Post caption",
+#       published_at: 2024-03-20 12:00:00
+#     }
+#   ]
+# }
+
+# To fetch next page:
+next_page = SocialMiner::Instagram.profile_posts(user_id: "user_id", cursor: result[:cursor]) do |posts_attrs, _|
+  ...
+end
+```
+
+To fetch comments on a specific post:
+
+```ruby
+mapper = SocialMiner::Instagram::CommentMapper
+
+result, cursor = SocialMiner::Instagram.post_comments(post_shortcode: "ABC123") do |comments_attrs, cursor|
+  [
+    comments_attrs.map { |comment_attrs| mapper.map(comment_attrs) },
+    cursor
+  ]
+end
+
+# Result contains:
+# {
+#   cursor: "next_page_token",
+#   records: [
+#     {
+#       social_id: "comment123",
+#       body: "Great post!",
+#       published_at: 2024-03-20 12:00:00
+#     }
+#   ]
+# }
+
+# To fetch next page of comments:
+next_page = SocialMiner::Instagram.post_comments(post_shortcode: "ABC123", cursor: result[:cursor]) do |comments_attrs, _|
+  ...
+end
+```
+
+# Custom Headers
+
+You can pass custom headers to the requests by passing a hash to the `request_headers` option:
+
+```ruby
+SocialMiner::Instagram.profile_info({ "Custom-Header" => "Header-Value" }, username: 'instagram')
+```
+
+# Custom Mapper
+
+You can use a custom mapper to transform the data into a custom structure.
+
+```ruby
+module CustomMapper
+  def map(attrs)
+    { id: attrs['id'] }
+  end
+
+  module_function :map
+end
+```
+
+Use your custom mapper:
+
+```ruby
+SocialMiner::Instagram.profile_info(username: "instagram") do |profile_attrs|
+  CustomMapper.map(profile_attrs)
+end
+```
+
+# Fetch all posts / comments
+
+```ruby
+class BackgroundJob
+  @@mapper = SocialMiner::Instagram::PostMapper
+
+  def perform(user_id, cursor: nil)
+    posts, cursor = SocialMiner::Instagram.profile_posts(user_id: user_id, cursor: cursor) do |posts_attrs, cursor|
+      [
+        posts_attrs.map { |post_attrs| @@mapper.map(post_attrs) },
+        cursor
+      ]
+    end
+
+    # Bulk import to DB
+    Post.import(posts)
+
+    perform_async(user_id, cursor) unless cursor.nil?
+  end
+end
+```
+
+### Default Mappers
+
+The gem includes several default mappers:
+
+- `ProfileMapper`: Maps user profile information
+  ```ruby
+  {
+    social_id: String,     # Instagram user ID
+    username: String,      # Instagram username
+    description: String,   # User bio
+    full_name: String,     # Full name
+    avatar_url: String,    # Profile picture URL
+    followers: Integer,    # Followers count
+    following: Integer     # Following count
+  }
+  ```
+
+- `PostMapper`: Maps post information
+  ```ruby
+  {
+    social_id: String,      # Instagram post ID
+    image_url: String,      # Post image URL
+    shortcode: String,      # Post shortcode
+    location_name: String,  # Location name (optional)
+    description: String,    # Post description
+    published_at: Time      # Post creation timestamp
+  }
+  ```
+
+- `CommentMapper`: Maps comment information
+  ```ruby
+  {
+    social_id: String,     # Comment ID
+    body: String,          # Comment text
+    published_at: Time     # Comment timestamp
+  }
+  ```
+
+# Requirements
+
+- Ruby 3.2+
+
+# License
+
+The gem is available as open source under the terms of the [MIT License](https://opensource.org/licenses/MIT).
+
+⚠️ **DISCLAIMER**
+
+This gem is provided for educational purposes only. Please note:
+
+1. This tool is not officially associated with Instagram or Meta Platforms, Inc.
+2. Using this gem might violate Instagram's Terms of Service.
+3. The author(s) of this gem:
+   - Take no responsibility for how you use this software
+   - Are not liable for any damages resulting from its use
+   - Are not responsible for any legal consequences that may arise from its use
+
+By using this gem, you acknowledge that:
+- You are solely responsible for complying with all applicable laws and terms of service
+- You will use this tool responsibly and at your own risk
+- You understand that Instagram's API changes may break this gem's functionality at any time

data/Rakefile

@@ -0,0 +1,12 @@
+# frozen_string_literal: true
+
+require "bundler/gem_tasks"
+require "rspec/core/rake_task"
+
+RSpec::Core::RakeTask.new(:spec)
+
+require "rubocop/rake_task"
+
+RuboCop::RakeTask.new
+
+task default: %i[spec rubocop]

data/lib/social_miner/helpers/hash.rb

@@ -0,0 +1,16 @@
+# frozen_string_literal: true
+
+module SocialMiner
+  module Helpers
+    module Hash
+      def fetch_nested(hash, *keys)
+        val = hash.fetch(keys.shift)
+        return val if keys.empty?
+
+        fetch_nested(val, *keys)
+      end
+
+      module_function :fetch_nested
+    end
+  end
+end

data/lib/social_miner/instagram/base.rb

@@ -0,0 +1,27 @@
+# frozen_string_literal: true
+
+require "json"
+
+require "uri"
+require "net/http"
+
+module SocialMiner
+  module Instagram
+    class Base
+      SOURCE_URL  = "https://www.instagram.com"
+
+      API_URL     = "#{SOURCE_URL}/api/v1".freeze
+      GRAPHQL_URL = "#{SOURCE_URL}/graphql/query/".freeze
+
+      DEFAULT_HEADERS = {
+        "X-IG-App-ID" => "936619743392459"
+      }.freeze
+
+      attr_reader :request_headers
+
+      def initialize(request_headers = {})
+        @request_headers = DEFAULT_HEADERS.merge(request_headers)
+      end
+    end
+  end
+end

data/lib/social_miner/instagram/comment_mapper.rb

@@ -0,0 +1,19 @@
+# frozen_string_literal: true
+
+module SocialMiner
+  module Instagram
+    module CommentMapper
+      def map(attrs)
+        published_at = Time.at(attrs["created_at"]) if attrs["created_at"]
+
+        {
+          social_id: attrs.fetch("id"),
+          body: attrs.fetch("text"),
+          published_at: published_at
+        }
+      end
+
+      module_function :map
+    end
+  end
+end

data/lib/social_miner/instagram/post_comments.rb

@@ -0,0 +1,45 @@
+# frozen_string_literal: true
+
+module SocialMiner
+  module Instagram
+    class PostComments < Base
+      DOC_ID = "8845758582119845"
+
+      def call(post_shortcode:, cursor: nil)
+        variables = { shortcode: post_shortcode }
+        variables[:after] = cursor if cursor
+
+        data = {
+          "variables" => { shortcode: post_shortcode }.to_json,
+          "doc_id" => DOC_ID
+        }
+
+        response = Net::HTTP.post_form(URI(GRAPHQL_URL), data)
+
+        case response
+        when Net::HTTPSuccess
+          response_data = JSON.parse(response.body)
+
+          edge_media = Helpers::Hash.fetch_nested(response_data,
+                                                  "data",
+                                                  "xdt_shortcode_media",
+                                                  "edge_media_to_parent_comment")
+
+          cursor = Helpers::Hash.fetch_nested(edge_media, "page_info", "end_cursor")
+                                .then { |data| JSON.parse(data) }
+                                .then { |data| data.fetch("server_cursor") }
+
+          records = edge_media.fetch("edges").map { |edge| edge.fetch("node") }
+
+          if block_given?
+            yield(records, cursor)
+          else
+            { records: records, cursor: cursor }
+          end
+        else
+          raise Net::HTTPError.new("Request Failed #{response.code}", response)
+        end
+      end
+    end
+  end
+end

data/lib/social_miner/instagram/post_mapper.rb

@@ -0,0 +1,30 @@
+# frozen_string_literal: true
+
+module SocialMiner
+  module Instagram
+    module PostMapper
+      def map(attrs)
+        published_at = Time.at(attrs["taken_at_timestamp"]) if attrs["taken_at_timestamp"]
+
+        {
+          social_id: attrs.fetch("id"),
+          image_url: attrs.fetch("display_url"),
+          shortcode: attrs.fetch("shortcode"),
+          location_name: attrs.dig("location", "name"),
+          description: description(attrs.dig("edge_media_to_caption", "edges")),
+          published_at: published_at
+        }
+      end
+
+      module_function :map
+
+      def description(edges)
+        edge = edges&.first
+        edge&.dig("node", "text")
+      end
+
+      module_function :description
+      private_class_method :description
+    end
+  end
+end

data/lib/social_miner/instagram/profile_info.rb

@@ -0,0 +1,32 @@
+# frozen_string_literal: true
+
+module SocialMiner
+  module Instagram
+    class ProfileInfo < Base
+      URL = "#{API_URL}/users/web_profile_info/".freeze
+
+      def call(username:)
+        uri = URI(URL)
+        uri.query = URI.encode_www_form([
+                                          [:username, username]
+                                        ])
+
+        response = Net::HTTP.get_response(uri, request_headers)
+
+        case response
+        when Net::HTTPSuccess
+          response_data = JSON.parse(response.body)
+          record = Helpers::Hash.fetch_nested(response_data, "data", "user")
+
+          if block_given?
+            yield(record)
+          else
+            record
+          end
+        else
+          raise Net::HTTPError.new("Request Failed #{response.code}", response)
+        end
+      end
+    end
+  end
+end

data/lib/social_miner/instagram/profile_mapper.rb

@@ -0,0 +1,21 @@
+# frozen_string_literal: true
+
+module SocialMiner
+  module Instagram
+    module ProfileMapper
+      def map(attrs)
+        {
+          social_id: attrs.fetch("id"),
+          description: attrs["biography"],
+          username: attrs.fetch("username"),
+          full_name: attrs.fetch("full_name"),
+          avatar_url: attrs.fetch("profile_pic_url"),
+          followers: Helpers::Hash.fetch_nested(attrs, "edge_followed_by", "count"),
+          following: Helpers::Hash.fetch_nested(attrs, "edge_follow", "count")
+        }
+      end
+
+      module_function :map
+    end
+  end
+end

data/lib/social_miner/instagram/profile_posts.rb

@@ -0,0 +1,44 @@
+# frozen_string_literal: true
+
+module SocialMiner
+  module Instagram
+    class ProfilePosts < Base
+      PER_PAGE = 12
+      QUERY_HASH = "69cba40317214236af40e7efa697781d"
+
+      def call(user_id:, cursor: nil)
+        uri = URI(GRAPHQL_URL)
+
+        variables = { id: user_id, first: PER_PAGE }
+        variables[:after] = cursor if cursor
+
+        uri.query = URI.encode_www_form([
+                                          [:query_hash, QUERY_HASH],
+                                          [:variables, variables.to_json]
+                                        ])
+
+        response = Net::HTTP.get_response(uri, request_headers)
+
+        case response
+        when Net::HTTPSuccess
+          response_data  = JSON.parse(response.body)
+          timeline_media = Helpers::Hash.fetch_nested(response_data,
+                                                      "data",
+                                                      "user",
+                                                      "edge_owner_to_timeline_media")
+
+          cursor  = Helpers::Hash.fetch_nested(timeline_media, "page_info", "end_cursor")
+          records = timeline_media.fetch("edges").map { |attrs| attrs.fetch("node") }
+
+          if block_given?
+            yield(records, cursor)
+          else
+            { records: records, cursor: cursor }
+          end
+        else
+          raise Net::HTTPError.new("Request Failed #{response.code}", response)
+        end
+      end
+    end
+  end
+end

data/lib/social_miner/version.rb

@@ -0,0 +1,5 @@
+# frozen_string_literal: true
+
+module SocialMiner
+  VERSION = "0.1.0"
+end

data/lib/social_miner.rb

@@ -0,0 +1,27 @@
+# frozen_string_literal: true
+
+require_relative "social_miner/version"
+
+require_relative "social_miner/helpers/hash"
+
+require_relative "social_miner/instagram/base"
+require_relative "social_miner/instagram/profile_info"
+require_relative "social_miner/instagram/profile_mapper"
+require_relative "social_miner/instagram/profile_posts"
+require_relative "social_miner/instagram/post_mapper"
+require_relative "social_miner/instagram/post_comments"
+require_relative "social_miner/instagram/comment_mapper"
+
+module SocialMiner
+  def Instagram.profile_info(request_headers = {}, **args, &)
+    Instagram::ProfileInfo.new(request_headers).call(**args, &)
+  end
+
+  def Instagram.profile_posts(request_headers = {}, **args, &)
+    Instagram::ProfilePosts.new(request_headers).call(**args, &)
+  end
+
+  def Instagram.post_comments(request_headers = {}, **args, &)
+    Instagram::PostComments.new(request_headers).call(**args, &)
+  end
+end

metadata

@@ -0,0 +1,63 @@
+--- !ruby/object:Gem::Specification
+name: social_miner
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- null0vector
+autorequire:
+bindir: exe
+cert_chain: []
+date: 2024-11-01 00:00:00.000000000 Z
+dependencies: []
+description:
+email:
+- root@null0vector.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- ".rspec"
+- ".rubocop.yml"
+- CHANGELOG.md
+- LICENSE.txt
+- README.md
+- Rakefile
+- lib/social_miner.rb
+- lib/social_miner/helpers/hash.rb
+- lib/social_miner/instagram/base.rb
+- lib/social_miner/instagram/comment_mapper.rb
+- lib/social_miner/instagram/post_comments.rb
+- lib/social_miner/instagram/post_mapper.rb
+- lib/social_miner/instagram/profile_info.rb
+- lib/social_miner/instagram/profile_mapper.rb
+- lib/social_miner/instagram/profile_posts.rb
+- lib/social_miner/version.rb
+homepage:
+licenses:
+- MIT
+metadata:
+  source_code_uri: https://github.com/null0vector/social_miner
+  changelog_uri: https://github.com/null0vector/social_miner/blob/main/CHANGELOG.md
+  rubygems_mfa_required: 'true'
+post_install_message:
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: 3.2.0
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubygems_version: 3.5.16
+signing_key:
+specification_version: 4
+summary: SocialMiner is a Ruby gem that provides a simple interface to fetch public
+  data from Instagram,     including user profiles, posts, and comments.
+test_files: []