kimurai_dynamic 1.4.1

data/Rakefile

@@ -0,0 +1,10 @@
+require "bundler/gem_tasks"
+require "rake/testtask"
+
+Rake::TestTask.new(:test) do |t|
+  t.libs << "test"
+  t.libs << "lib"
+  t.test_files = FileList["test/**/*_test.rb"]
+end
+
+task :default => :test

data/bin/console

@@ -0,0 +1,14 @@
+#!/usr/bin/env ruby
+
+require "bundler/setup"
+require "kimurai"
+
+# You can add fixtures and/or initialization code here to make experimenting
+# with your gem easier. You can also use a different console, if you like.
+
+# (If you use this, don't forget to add pry to your Gemfile!)
+# require "pry"
+# Pry.start
+
+require "irb"
+IRB.start(__FILE__)

data/bin/setup

@@ -0,0 +1,8 @@
+#!/usr/bin/env bash
+set -euo pipefail
+IFS=$'\n\t'
+set -vx
+
+bundle install
+
+# Do any other automated setup that you need to do here

data/exe/kimurai

@@ -0,0 +1,6 @@
+#!/usr/bin/env ruby
+
+require 'kimurai'
+require 'kimurai/cli'
+
+Kimurai::CLI.start(ARGV)

data/kimurai.gemspec

@@ -0,0 +1,48 @@
+
+lib = File.expand_path("../lib", __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require "kimurai/version"
+
+Gem::Specification.new do |spec|
+  spec.name          = "kimurai_dynamic"
+  spec.version       = Kimurai::VERSION
+  spec.authors       = ["Mehmet Celik"]
+  spec.email         = ['mehmetcelik4@gmail.com']
+
+  spec.summary       = "Modern web scraping framework written in Ruby and based on Capybara/Nokogiri, with dynamic crawl option"
+  spec.homepage      = "https://github.com/MehmetCelik4/kimuraframework"
+  spec.license       = "MIT"
+
+  # Specify which files should be added to the gem when it is released.
+  # The `git ls-files -z` loads the files in the RubyGem that have been added into git.
+  spec.files         = Dir.chdir(File.expand_path('..', __FILE__)) do
+    `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features)/}) }
+  end
+  spec.bindir        = "exe"
+  spec.executables   = "kimurai"
+  spec.require_paths = ["lib"]
+  spec.required_ruby_version = ">= 2.5.0"
+
+  spec.add_dependency "thor"
+  spec.add_dependency "cliver"
+  spec.add_dependency "activesupport"
+  spec.add_dependency "murmurhash3"
+  spec.add_dependency "nokogiri"
+
+  spec.add_dependency "capybara", ">= 2.15", "< 4.0"
+  spec.add_dependency "capybara-mechanize"
+  spec.add_dependency "poltergeist"
+  spec.add_dependency "selenium-webdriver"
+
+  spec.add_dependency "headless"
+  spec.add_dependency "pmap"
+
+  spec.add_dependency "whenever"
+
+  spec.add_dependency "rbcat", "~> 0.2"
+  spec.add_dependency "pry"
+
+  spec.add_development_dependency "bundler", "~> 1.16"
+  spec.add_development_dependency "rake", "~> 10.0"
+  spec.add_development_dependency "minitest", "~> 5.0"
+end

data/lib/kimurai/automation/deploy.yml

@@ -0,0 +1,54 @@
+---
+- hosts: all
+  vars:
+    rbenv_root_path: /home/{{ ansible_user_id }}/.rbenv
+    rbenv_shims_path: "{{ rbenv_root_path }}/shims"
+    repo_url:
+    repo_name:
+    repo_key_path:
+
+  tasks:
+    - name: Copy custom git ssh key to /tmp/private_key (if provided)
+      when: repo_key_path is not none
+      copy:
+        src: "{{ repo_key_path }}"
+        dest: /tmp/private_key
+        mode: 0600
+
+    - name: Clone/pull project repo to ~/{{ repo_name }} user directory (using ssh-agent forwarding or https)
+      when: repo_key_path is none
+      git:
+        repo: "{{ repo_url }}"
+        dest: "~/{{ repo_name }}"
+        force: true
+        accept_hostkey: true
+
+    - name: Clone/pull project repo to ~/{{ repo_name }} user directory (using custom git ssh key)
+      when: repo_key_path is not none
+      git:
+        repo: "{{ repo_url }}"
+        dest: "~/{{ repo_name }}"
+        force: true
+        accept_hostkey: true
+        key_file: /tmp/private_key
+
+    - name: Delete custom git ssh key from /tmp/private_key (if provided)
+      when: repo_key_path is not none
+      file:
+        state: absent
+        path: /tmp/private_key
+
+    - name: Run bundle install
+      command: bundle install
+      args:
+        chdir: ~/{{ repo_name }}
+      environment:
+        PATH: "{{ rbenv_root_path }}/bin:{{ rbenv_root_path }}/shims:{{ ansible_env.PATH }}"
+
+    - name: Run whenever to update crontab
+      command: whenever --update-crontab
+      args:
+        chdir: ~/{{ repo_name }}
+      environment:
+        PATH: "{{ rbenv_root_path }}/bin:{{ rbenv_root_path }}/shims:{{ ansible_env.PATH }}"
+

data/lib/kimurai/automation/setup/chromium_chromedriver.yml

@@ -0,0 +1,26 @@
+---
+- name: Install chromium browser
+  apt:
+    pkg: chromium-browser
+    state: present
+
+- name: Get current chromedriver version
+  shell: chromedriver --version
+  args:
+    executable: /bin/bash
+  register: current_chromedriver_version
+  changed_when: false
+  ignore_errors: true
+
+- name: Install unzip tool to unarchive chromedriver archive
+  apt:
+    pkg: unzip
+    state: present
+
+- name: Download chromedriver binary archive and unarchive it to /usr/local/bin
+  unarchive:
+    src: https://chromedriver.storage.googleapis.com/{{ chromedriver }}/chromedriver_linux64.zip
+    dest: /usr/local/bin
+    remote_src: true
+    mode: a+x
+  when: chromedriver not in current_chromedriver_version.stdout_lines

data/lib/kimurai/automation/setup/firefox_geckodriver.yml

@@ -0,0 +1,20 @@
+---
+- name: Install firefox
+  apt:
+    pkg: firefox
+    state: present
+
+- name: Get current geckodriver version
+  shell: geckodriver --version
+  args:
+    executable: /bin/bash
+  register: current_geckodriver_version
+  changed_when: false
+  ignore_errors: true
+
+- name: Download geckodriver binary archive and unarchive it to /usr/local/bin
+  unarchive:
+    src: https://github.com/mozilla/geckodriver/releases/download/v{{ geckodriver }}/geckodriver-v{{ geckodriver }}-linux64.tar.gz
+    dest: /usr/local/bin
+    remote_src: true
+  when: geckodriver not in current_geckodriver_version.stdout

data/lib/kimurai/automation/setup/phantomjs.yml

@@ -0,0 +1,33 @@
+---
+- name: Install dependencies for PhantomJS
+  apt:
+    pkg: "{{ item }}"
+    state: present
+  with_items:
+    - chrpath
+    - libxft-dev
+    - libfreetype6
+    - libfreetype6-dev
+    - libfontconfig1
+    - libfontconfig1-dev
+
+- name: Get current phantomjs version
+  shell: phantomjs -v
+  args:
+    executable: /bin/bash
+  register: current_phantomjs_version
+  changed_when: false
+  ignore_errors: true
+
+- name: Download phantomJS archive and unarchive it to /usr/local/lib
+  unarchive:
+    src: https://bitbucket.org/ariya/phantomjs/downloads/phantomjs-{{ phantomjs }}-linux-x86_64.tar.bz2
+    dest: /usr/local/lib
+    remote_src: true
+  when: phantomjs not in current_phantomjs_version.stdout
+
+- name: Link PhantomJS binary to /usr/local/bin/phantomjs
+  file:
+    src: /usr/local/lib/phantomjs-{{ phantomjs }}-linux-x86_64/bin/phantomjs
+    dest: /usr/local/bin/phantomjs
+    state: link

data/lib/kimurai/automation/setup/ruby_environment.yml

@@ -0,0 +1,124 @@
+---
+- name: Install dependencies for ruby-build
+  become: true
+  apt:
+    pkg: "{{ item }}"
+    state: present
+  with_items:
+    - zlib1g-dev
+    - build-essential
+    - libssl-dev
+    - libreadline-dev
+    - libreadline6-dev
+    - libyaml-dev
+    - libxml2-dev
+    - libxslt1-dev
+    - libcurl4-openssl-dev
+    - libffi-dev
+
+- name: Clone Rbenv repository to the {{ ansible_user_id }} user directory
+  git:
+    repo: https://github.com/sstephenson/rbenv.git
+    dest: "{{ rbenv_root_path }}"
+
+- name: Clone ruby-build repo to the {{ ansible_user_id }} user directory
+  git:
+    repo: https://github.com/sstephenson/ruby-build.git
+    dest: "{{ rbenv_root_path }}/plugins/ruby-build"
+
+- name: Add Rbenv path to the .bashrc
+  lineinfile:
+    dest: ~/.bashrc
+    regexp: '^export PATH="\$HOME\/\.rbenv'
+    line: export PATH="$HOME/.rbenv/bin:$PATH"
+    state: present
+
+- name: Add Rbenv init to the .bashrc
+  lineinfile:
+    dest: ~/.bashrc
+    regexp: '^eval "\$\(rbenv'
+    line: eval "$(rbenv init -)"
+    state: present
+
+- name: Check if desired Ruby version already installed
+  stat:
+    path: "{{ ruby_versions_path }}/{{ ruby }}"
+  register: ruby_present
+
+- name: Install desired Ruby version using ruby-build (this can take a while)
+  command: rbenv install {{ ruby }}
+  when: not ruby_present.stat.exists
+  environment:
+    CONFIGURE_OPTS: "--disable-install-doc"
+    PATH: "{{ rbenv_root_path }}/bin:{{ rbenv_root_path }}/shims:{{ ansible_env.PATH }}"
+
+- name: Get current Ruby version
+  command: "ruby -v"
+  register: current_ruby_version
+  changed_when: false
+  ignore_errors: true
+  environment:
+    PATH: "{{ rbenv_root_path }}/bin:{{ rbenv_root_path }}/shims:{{ ansible_env.PATH }}"
+
+- name: Set desired Ruby version as a global version
+  command: "rbenv global {{ ruby }}"
+  when: ruby not in current_ruby_version.stdout
+  environment:
+    PATH: "{{ rbenv_root_path }}/bin:{{ rbenv_root_path }}/shims:{{ ansible_env.PATH }}"
+  register: set_ruby
+
+- name: Execute `rbenv rehash` command
+  command: rbenv rehash
+  when: set_ruby.changed
+  environment:
+    PATH: "{{ rbenv_root_path }}/bin:{{ rbenv_root_path }}/shims:{{ ansible_env.PATH }}"
+
+- name: Create ~/.gemrc file to skip docs
+  copy:
+    dest: ~/.gemrc
+    content: "gem: --no-ri --no-rdoc"
+
+- name: Create ~/.bundle directory
+  file:
+    dest: ~/.bundle
+    state: directory
+
+- name: Create ~/.bundle/config file with default settings `BUNDLE_GIT__ALLOW_INSECURE true` and `BUNDLE_JOBS 4`
+  copy:
+    dest: ~/.bundle/config
+    content: |
+      BUNDLE_GIT__ALLOW_INSECURE: "true"
+      BUNDLE_JOBS: "4"
+
+- name: Check if Bundler gem installed
+  stat:
+    path: "{{ ruby_versions_path }}/{{ ruby }}/bin/bundler"
+  register: bundler_gem_present
+
+- name: Install Bundler gem
+  command: gem install bundler
+  when: not bundler_gem_present.stat.exists
+  environment:
+    PATH: "{{ rbenv_root_path }}/bin:{{ rbenv_root_path }}/shims:{{ ansible_env.PATH }}"
+
+- name: Check if Whenever gem installed
+  stat:
+    path: "{{ ruby_versions_path }}/{{ ruby }}/bin/whenever"
+  register: whenever_gem_present
+
+- name: Install Whenever gem
+  command: gem install whenever
+  when: not whenever_gem_present.stat.exists
+  environment:
+    PATH: "{{ rbenv_root_path }}/bin:{{ rbenv_root_path }}/shims:{{ ansible_env.PATH }}"
+
+- name: Check if Kimurai gem installed
+  stat:
+    path: "{{ ruby_versions_path }}/{{ ruby }}/bin/kimurai"
+  register: kimurai_gem_present
+
+- name: Install Kimurai gem
+  command: gem install kimurai
+  when: not kimurai_gem_present.stat.exists
+  environment:
+    PATH: "{{ rbenv_root_path }}/bin:{{ rbenv_root_path }}/shims:{{ ansible_env.PATH }}"

data/lib/kimurai/automation/setup.yml

@@ -0,0 +1,45 @@
+---
+- hosts: all
+  vars:
+    ruby: 2.5.3
+    rbenv_root_path: /home/{{ ansible_user_id }}/.rbenv
+    rbenv_shims_path: "{{ rbenv_root_path }}/shims"
+    ruby_versions_path: "{{ rbenv_root_path }}/versions"
+    # check latest here http://phantomjs.org/download.html
+    phantomjs: 2.1.1
+    # check latest here https://github.com/mozilla/geckodriver/releases/
+    geckodriver: 0.23.0
+    # check latest here https://sites.google.com/a/chromium.org/chromedriver/downloads
+    chromedriver: 2.44
+
+  tasks:
+    - name: Update apt cache
+      become: true
+      apt: update_cache=yes cache_valid_time=86400
+
+    - name: Install base packages
+      become: true
+      apt:
+        pkg: "{{ item }}"
+        state: present
+      with_items:
+        - git
+        - xvfb
+        - libsqlite3-dev
+        - sqlite3
+        - mongodb-clients
+        - mysql-client
+        - libmysqlclient-dev
+        - postgresql-client
+        - libpq-dev
+
+    - import_tasks: setup/ruby_environment.yml
+
+    - import_tasks: setup/phantomjs.yml
+      become: true
+
+    - import_tasks: setup/firefox_geckodriver.yml
+      become: true
+
+    - import_tasks: setup/chromium_chromedriver.yml
+      become: true

data/lib/kimurai/base/saver.rb

@@ -0,0 +1,106 @@
+require 'json'
+require 'csv'
+
+module Kimurai
+  class Base
+    class Saver
+      attr_reader :format, :path, :position, :append
+
+      def initialize(path, format:, position: true, append: false)
+        unless %i(json pretty_json jsonlines csv).include?(format)
+          raise "SimpleSaver: wrong type of format: #{format}"
+        end
+
+        @path = path
+        @format = format
+        @position = position
+        @index = 0
+        @append = append
+        @mutex = Mutex.new
+      end
+
+      def save(item)
+        @mutex.synchronize do
+          @index += 1
+          item[:position] = @index if position
+
+          case format
+          when :json
+            save_to_json(item)
+          when :pretty_json
+            save_to_pretty_json(item)
+          when :jsonlines
+            save_to_jsonlines(item)
+          when :csv
+            save_to_csv(item)
+          end
+        end
+      end
+
+      private
+
+      def save_to_json(item)
+        data = JSON.generate([item])
+
+        if @index > 1 || append && File.exists?(path)
+          file_content = File.read(path).sub(/\}\]\Z/, "\}\,")
+          File.open(path, "w") do |f|
+            f.write(file_content + data.sub(/\A\[/, ""))
+          end
+        else
+          File.open(path, "w") { |f| f.write(data) }
+        end
+      end
+
+      def save_to_pretty_json(item)
+        data = JSON.pretty_generate([item])
+
+        if @index > 1 || append && File.exists?(path)
+          file_content = File.read(path).sub(/\}\n\]\Z/, "\}\,\n")
+          File.open(path, "w") do |f|
+            f.write(file_content + data.sub(/\A\[\n/, ""))
+          end
+        else
+          File.open(path, "w") { |f| f.write(data) }
+        end
+      end
+
+      def save_to_jsonlines(item)
+        data = JSON.generate(item)
+
+        if @index > 1 || append && File.exists?(path)
+          File.open(path, "a") { |file| file.write("\n" + data) }
+        else
+          File.open(path, "w") { |file| file.write(data) }
+        end
+      end
+
+      def save_to_csv(item)
+        data = flatten_hash(item)
+
+        if @index > 1 || append && File.exists?(path)
+          CSV.open(path, "a+", force_quotes: true) do |csv|
+            csv << data.values
+          end
+        else
+          CSV.open(path, "w", force_quotes: true) do |csv|
+            csv << data.keys
+            csv << data.values
+          end
+        end
+      end
+
+      def flatten_hash(hash)
+        hash.each_with_object({}) do |(k, v), h|
+          if v.is_a? Hash
+            flatten_hash(v).map { |h_k, h_v| h["#{k}.#{h_k}"] = h_v }
+          else
+            h[k&.to_s] = v
+          end
+        end
+      end
+    end
+  end
+end
+
+

data/lib/kimurai/base/storage.rb

@@ -0,0 +1,54 @@
+module Kimurai
+  class Base
+    class Storage
+      attr_reader :database
+
+      def initialize
+        @mutex = Mutex.new
+        @database = {}
+      end
+
+      def all(scope = nil)
+        @mutex.synchronize do
+          scope ? database.fetch(scope, []) : database
+        end
+      end
+
+      def include?(scope, value)
+        @mutex.synchronize do
+          database[scope] ||= []
+          database[scope].include?(value)
+        end
+      end
+
+      def add(scope, value)
+        @mutex.synchronize do
+          database[scope] ||= []
+          if value.kind_of?(Array)
+            database[scope] += value
+            database[scope].uniq!
+          else
+            database[scope].push(value) unless database[scope].include?(value)
+          end
+        end
+      end
+
+      ###
+
+      def unique?(scope, value)
+        @mutex.synchronize do
+          database[scope] ||= []
+          database[scope].include?(value) ? false : database[scope].push(value) and true
+        end
+      end
+
+      ###
+
+      def clear!
+        @mutex.synchronize do
+          @database = {}
+        end
+      end
+    end
+  end
+end