foxit 0.1.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: 541b370e85c872627d4bff2419a6af1d8ce922c8e19363218aad0f6642b435ab
+  data.tar.gz: 74738a7174d332d718573188510a6e4744f8b08b7cb66735ae758fadb65dfd1c
+SHA512:
+  metadata.gz: 4dcfdbcf7d13ec79d4501298fe49379b07fff9736829a42f01ecaee043aa0ccfde9a1bd6861c8ea63bb0824cf6f134a1ec81e9d5f83956388a86420e4d872ca9
+  data.tar.gz: e597208dcabad4affd0ff242bbcec7cddc7ae85afb0f499193d85b82dbf9885a0cb32b2719795907d60608c21c55671ea5daf32c3356ccc4e24b0a2c0f18f974

data/.gitignore

@@ -0,0 +1,6 @@
+*.gem
+
+/scratch
+/notebooks
+/examples
+/.vscode

data/Gemfile

@@ -0,0 +1,3 @@
+source "https://rubygems.org"
+
+gemspec

data/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2018 Lachlan Taylor
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

data/README.md

@@ -0,0 +1,136 @@
+# Kitsu API Wrapper
+Unofficial Ruby wrapper for [Kitsu][kitsu] API ([documentation][api]).
+
+Note: this is the initial version and was written primarily to store data locally on which to train recommendation models. Hence, it works well for the task it was created but there are a few gaps that could be included without too much effort. 
+
+
+## Installation
+```
+gem install foxit
+```
+
+## Usage
+A few quick examples are detailed below:
+
+### Retrieving Data
+Get an anime item by id. 
+```ruby
+api = Foxit::API.new()
+result = api.get_anime_by_id(1)
+```
+
+This returns the full json response; however, an object based result can be returned as per the following (planned to be returned as default)
+```ruby
+item = Anime.new(result['data'])
+```
+
+Similarly, getting a users library. This returns a list of entries from the `'data'` attribute of the json returned. 
+```ruby
+result = api.get_library_by_id(1)
+```
+
+
+### Storing Data with MongoDB
+If you don't have mongodb installed you can download the free community version from [here][mongodb]. Once installed, you need to start the service via terminal:
+```
+mongod --port 27017 --dbpath /path/to/db
+```
+Note: you can stop it by pressing `ctrl+c` when the terminal is active or by using
+```
+mongod --dbpath /path/to/db --shutdown
+```
+
+Once you have mongodb running you can retreive and store data by using the following functions.
+
+**Please use responsibly!** Library results are especially expensive as they require a secondary call to lookup the media_id from the library item id that is returned in the user library request. In other words, for the benefit of everyone, don't punish the API with hundreds of calls.
+
+```ruby
+etl = Foxit::ETL.new()
+etl.get_anime(1..100)  # stores anime item results from media_id 1 to 100
+etl.get_libraries(1..25)  # stores user library results from user_id 1 to 25
+```
+
+#### MongoDB Library Item
+Current fields obtained from Kitsu API. There are more available and could easilty be added; however, this was all I needed for working on the recommendation models.
+```
+{
+  '_id': ObjectId('5b56ad7dafd2a496c0faa495'),
+  'media_id': 1376,       # id for anime, e.g., this one relates to 'Code Geass: Lelouch of the Rebellion'
+  'rating': 17,           # user rating [1-20]; None if not rated.
+  'record_id': 18344699,  # library item id; used to return the actual media_id
+  'status': 'completed',  # media viewing status: {completed, planned, watching, hold, dropped}
+  'type': 'anime',        # media type: {Anime, Manga, Drama?}.
+  'user_id': 1            # user id
+}
+```
+
+
+#### MongoDB Anime Item
+
+```
+{ 
+  '_id': ObjectId('5b570e4eafd2a41db810927b'),
+  'avg_rating': 84.36,
+  'end_date': '1999-04-24',
+  'id': 1,
+  'n_favourites': 0,
+  'n_users': 71919,
+  'nsfw': False,
+  'rank_popularity': 15,
+  'rank_rating': 27,
+  'rating_freq': { '10': 415,
+                   '11': 27,
+                   '12': 1700,
+                   '13': 53,
+                   '14': 3878,
+                   '15': 139,
+                   '16': 5255,
+                   '17': 255,
+                   '18': 6261,
+                   '19': 238,
+                   '2': 1566,
+                   '20': 20974,
+                   '3': 33,
+                   '4': 351,
+                   '5': 15,
+                   '6': 115,
+                   '7': 16,
+                   '8': 1545,
+                   '9': 22},
+  'showtype': 'TV',
+  'slug': 'cowboy-bebop',
+  'start_date': '1998-04-03',
+  'subtype': 'TV',
+  'synopsis': 'In the year 2071, humanity has colonized several of the planets '
+              'and moons of the solar system leaving the now uninhabitable '
+              'surface of planet Earth behind. The Inter Solar System Police '
+              'attempts to keep peace in the galaxy, aided in part by outlaw '
+              'bounty hunters, referred to as "Cowboys". The ragtag team '
+              'aboard the spaceship Bebop are two such individuals.\r\n'
+              'Mellow and carefree Spike Spiegel is balanced by his '
+              'boisterous, pragmatic partner Jet Black as the pair makes a '
+              'living chasing bounties and collecting rewards. Thrown off '
+              'course by the addition of new members that they meet in their '
+              'travels�Ein, a genetically engineered, highly intelligent Welsh '
+              'Corgi; femme fatale Faye Valentine, an enigmatic trickster with '
+              'memory loss; and the strange computer whiz kid Edward Wong�the '
+              "crew embarks on thrilling adventures that unravel each member's "
+              'dark and mysterious past little by little. \r\n'
+              'Well-balanced with high density action and light-hearted '
+              'comedy, Cowboy Bebop is a space Western classic and an homage '
+              'to the smooth and improvised music it is named after.  \r\n'
+              '[Written by MAL Rewrite]',
+  'title': 'Cowboy Bebop'
+}
+```
+
+
+## Further Work
+- Add some more detail/examples to README
+- Extend on functionality/usability for benefit of other users.
+
+
+
+[kitsu]: https://kitsu.io
+[api]: https://kitsu.docs.apiary.io/
+[mongodb]: https://www.mongodb.com/download-center#community

data/foxit.gemspec

@@ -0,0 +1,29 @@
+
+lib = File.expand_path("../lib", __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+# require "foxit/version"
+
+Gem::Specification.new do |spec|
+  spec.name          = "foxit"
+  # spec.version       = Foxit::VERSION
+  spec.version       = "0.1.1"
+  spec.authors       = ["Lachlan Taylor"]
+  spec.email         = ["lachlanbtaylor@gmail.com"]
+
+  spec.summary       = %q{Unofficial Kitsu API wrapper with MongoDB capability.}
+  # spec.description   = %q{Write a longer description or delete this line.}
+  spec.homepage      = "https://github.com/rokkuran/foxit"
+  spec.license       = "MIT"
+
+  # Specify which files should be added to the gem when it is released.
+  # The `git ls-files -z` loads the files in the RubyGem that have been added into git.
+  spec.files         = Dir.chdir(File.expand_path('..', __FILE__)) do
+    `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features)/}) }
+  end
+
+  spec.require_paths = ["lib"]
+
+  spec.add_dependency 'json', '~> 2.1.0', '>= 2.0.4'
+  spec.add_dependency 'addressable', '~> 2.5.2', '>= 2.5.2'
+  spec.add_dependency 'mongo', '~> 2.5.1', '>= 2.5.1'
+end

data/lib/foxit/api.rb

@@ -0,0 +1,179 @@
+require_relative 'objects'
+
+require 'net/http'
+require 'json'
+require 'addressable/uri'
+
+
+
+module Foxit
+
+  class API
+
+    def initialize
+      @root = "https://kitsu.io/api/edge/"
+    end
+  
+  
+    def build_library_url id, type='Anime', status='completed', limit=500
+      uri = Addressable::URI.parse("#{@root}library-entries")
+  
+      uri_query = {
+        "filter[user_id]" => id,
+        "filter[media_type]" => type,
+        "filter[status]" => status,
+        "page[limit]" => limit
+      }
+      uri.query_values = uri_query
+      
+      uri.to_s
+    end
+  
+    
+    def build_media_url entry_id
+      "#{@root}/library-entries/#{entry_id}/relationships/media"
+    end
+  
+  
+    def get_result url
+      response = Net::HTTP.get(URI.parse(url))
+      JSON.parse(response)
+    end
+    
+  
+    def get_library_by_url url, entries=[]
+  
+      result = self.get_result(url)
+      entries += result['data']
+    
+      if result['links'].key?('next')
+        # recursion to retrieve additional results that have ben paginated
+        result = self.get_library_by_url(result['links']['next'], entries)
+      else
+        return entries
+      end
+    end
+  
+  
+    def get_library_by_id user_id
+      url = self.build_library_url(user_id)
+      self.get_library_by_url(url)
+    end
+  
+  
+    def get_media_relationship_by_id entry_id
+      # adding memoisation to stop requesting same media docs
+      # TODO: extend to items already existing in db?
+      @get_media_relationship_by_id ||= {}
+      return @get_media_relationship_by_id[entry_id] if @get_media_relationship_by_id.key?(entry_id)
+  
+      url = self.build_media_url(entry_id)
+      self.get_result(url)
+    end
+  
+  
+    def batch_get_results ids, fn, max_threads
+      """
+      ids: ids of results returned: user_id | library_entry_id in this case.
+      fn: function name to use to return results (need to use symbol method name)
+          e.g. :get_result
+      max_threads: maximum number of active threads.
+      """
+  
+      results = {}
+      threads = []
+  
+      ids.each do |id|
+  
+        if Thread.list.count % max_threads != 0
+          thread = Thread.new do
+            # adding lock slows down considerably shouldn't matter as results are written to hash?
+            results[id] = send(fn, id)
+          end
+          threads << thread
+        else
+          # wait for open threads to finish before starting new one
+          threads.each(&:join)
+  
+          thread = Thread.new do
+            results[id] = send(fn, id)
+          end
+          threads << thread
+        end
+  
+      end
+  
+      threads.each(&:join)
+  
+      results
+    end
+  
+    # TODO: should probably assign to a @variable
+    def batch_get_libraries user_ids, max_threads=200
+  
+      all_library_entries = []
+      user_libraries = self.batch_get_results(user_ids, :get_library_by_id, max_threads)
+    
+      user_libraries.each do |user_id, library|
+    
+        media_ids = []
+        library.map { |entry| media_ids << entry['id'] }
+        media_results = self.batch_get_results(media_ids, :get_media_relationship_by_id, max_threads)
+        
+        library.each do |entry|
+          all_library_entries << LibraryItem.new(user_id, entry, media_results[entry['id']])
+        end
+    
+      end
+    
+      all_library_entries
+    end
+  
+    
+    def get_user_library
+      # TODO: return user library using single id call to batch_get_library
+    end
+    
+  
+    def batch_get_libraries_docs user_ids, max_threads=200
+      all_library_entries = self.batch_get_libraries(user_ids, max_threads)
+      
+      docs = []
+      all_library_entries.map { |entry| docs << entry.to_hash }
+  
+      docs
+    end
+  
+  
+    def build_anime_url id
+      "#{@root}/anime/#{id}"
+    end
+  
+  
+    def get_anime_by_id id
+      url = build_anime_url(id)
+      self.get_result(url)
+    end
+  
+  
+    def batch_get_anime anime_ids, max_threads=200
+      results = self.batch_get_results(anime_ids, :get_anime_by_id, max_threads)
+  
+      anime_items = []
+      results.each do |id, result|
+        unless result.key?('errors')
+          anime_items << Anime.new(result['data'])
+        end
+      end
+  
+      anime_items
+    end
+  
+    def get_anime_documents anime_ids, max_threads=200
+      anime_items = self.batch_get_anime(anime_ids, max_threads)
+      self.objects_to_hash(anime_items)
+    end
+  
+  end
+
+end

data/lib/foxit/etl.rb

@@ -0,0 +1,43 @@
+require_relative 'api'
+
+require 'mongo'
+Mongo::Logger.logger.level = Logger::WARN
+
+
+
+module Foxit
+
+  class ETL
+    attr_reader :kitsu, :client
+  
+    def initialize db_name: 'kitsu', host: ['127.0.0.1:27017']
+      @kitsu = API.new()
+      @client = Mongo::Client.new(host, {database: db_name})
+    end
+  
+    def insert_many_docs collection_name, docs
+      begin
+        puts "inserting..."
+        result = @client[collection_name].insert_many(docs)
+        puts "records inserted: #{result.inserted_count}"
+      rescue StandardError => e
+        puts "error: #{e}"
+      end
+      puts "complete.\n"
+    end
+    
+    
+    def get_libraries user_ids
+      docs = @kitsu.batch_get_libraries_docs(user_ids)
+      insert_many_docs('library', docs)
+    end
+    
+    
+    def get_anime media_ids
+      docs = @kitsu.get_anime_documents(media_ids)
+      insert_many_docs('anime', docs)
+    end
+  
+  end
+    
+end

data/lib/foxit/helpers.rb

@@ -0,0 +1,13 @@
+class Helpers
+  def to_hash
+    hash = {}
+    instance_variables.each {|x| hash[x.to_s.delete("@")] = instance_variable_get(x)}
+    hash
+  end
+
+  def objects_to_hash obj_array
+    docs = []
+    obj_array.map { |obj| docs << obj.to_hash }
+    docs
+  end
+end

data/lib/foxit/objects.rb

@@ -0,0 +1,50 @@
+require_relative 'helpers'
+
+
+
+class LibraryItem < Helpers
+  attr_reader :user_id, :record_id, :status, :rating, :media_id, :type
+
+  def initialize user_id, entry_result, media_result
+    @user_id = user_id.to_i
+    @record_id = entry_result['id'].to_i
+    @status = entry_result['attributes']['status']
+
+    rating = entry_result['attributes']['ratingTwenty']
+    @rating = rating.nil? ? nil : rating.to_i
+    
+    @media_id = media_result['data']['id'].to_i
+    @type = media_result['data']['type']
+  end
+end
+
+
+class Anime < Helpers
+  attr_reader :id, :slug, :synopsis, :title, :avg_rating, :rating_freq, :n_users,
+    :n_favourites, :start_date, :end_date, :rank_popularity, :rank_rating, :subtype,
+    :nsfw; :showtype
+
+  def initialize data
+    attributes = data['attributes']
+    @id = data['id'].to_i
+    @slug = attributes['slug']
+    @synopsis = attributes['synopsis']
+    @title = attributes['canonicalTitle']
+    @avg_rating = attributes['averageRating'].to_f  # TODO: need to handle nil values?
+    
+    rf_int = {}
+    # mongodb needs string keys anyway, so k.to_i redundant...
+    attributes['ratingFrequencies'].each_pair { | k, v | rf_int[k.to_i] = v.to_i }
+    @rating_freq = rf_int
+
+    @n_users = attributes['userCount'].to_i
+    @n_favourites = attributes['favouritesCount'].to_i
+    @start_date = attributes['startDate']
+    @end_date = attributes['endDate']
+    @rank_popularity = attributes['popularityRank'].to_i
+    @rank_rating = attributes['ratingRank'].to_i
+    @subtype = attributes['subtype']
+    @showtype = attributes['showType']
+    @nsfw = attributes['nsfw']  # TODO: convert to bool?
+  end
+end

data/lib/foxit.rb

@@ -0,0 +1,11 @@
+require "foxit/objects"
+require "foxit/helpers"
+
+
+module Foxit
+  VERSION = "0.1.0"
+
+  require "foxit/api"
+  require "foxit/etl"
+
+end

metadata

@@ -0,0 +1,114 @@
+--- !ruby/object:Gem::Specification
+name: foxit
+version: !ruby/object:Gem::Version
+  version: 0.1.1
+platform: ruby
+authors:
+- Lachlan Taylor
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2018-07-26 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: json
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 2.1.0
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 2.0.4
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 2.1.0
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 2.0.4
+- !ruby/object:Gem::Dependency
+  name: addressable
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 2.5.2
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 2.5.2
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 2.5.2
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 2.5.2
+- !ruby/object:Gem::Dependency
+  name: mongo
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 2.5.1
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 2.5.1
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 2.5.1
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 2.5.1
+description: 
+email:
+- lachlanbtaylor@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- Gemfile
+- LICENSE
+- README.md
+- foxit.gemspec
+- lib/foxit.rb
+- lib/foxit/api.rb
+- lib/foxit/etl.rb
+- lib/foxit/helpers.rb
+- lib/foxit/objects.rb
+homepage: https://github.com/rokkuran/foxit
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.7.7
+signing_key: 
+specification_version: 4
+summary: Unofficial Kitsu API wrapper with MongoDB capability.
+test_files: []