crawl_station 0.1.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: f6d109fe2c385fdce5228dc5804145edb350025f
+  data.tar.gz: a2ccadba7a2f6965a49c7bca5e0cb9f46739dc6d
+SHA512:
+  metadata.gz: 68440f18e400e3800b08363973fd989210b2c06ab74888b8f5827085fb01008f76907af84c4bb3e7b2c6221f5eadac31fdf0cc150cf086a9d5c6d16798aaa48e
+  data.tar.gz: 67253f586d2e836f1265126af5ba999da119a66462fb0957372a0e5e41f10803c27062489325bdb362ee8993d31c2ad7a9bb41807c5aa5e7ceb48e7a5f0d71bb

data/.codeclimate.yml

@@ -0,0 +1,24 @@
+---
+engines:
+  duplication:
+    enabled: true
+    config:
+      languages:
+      - ruby
+      - javascript
+      - python
+      - php
+  fixme:
+    enabled: true
+  rubocop:
+    enabled: true
+ratings:
+  paths:
+  - "**.inc"
+  - "**.js"
+  - "**.jsx"
+  - "**.module"
+  - "**.py"
+  - "**.rb"
+exclude_paths:
+- spec/

data/.gitignore

@@ -0,0 +1,10 @@
+/.bundle/
+/.yardoc
+/Gemfile.lock
+/_yardoc/
+/coverage/
+/doc/
+/pkg/
+/spec/reports/
+/tmp/
+.byebug_history

data/.rspec

@@ -0,0 +1,2 @@
+--format documentation
+--color

data/.rubocop.yml

@@ -0,0 +1,98 @@
+AllCops:
+  TargetRubyVersion: 2.3
+  # RuboCop has a bunch of cops enabled by default. This setting tells RuboCop
+  # to ignore them, so only the ones explicitly set in this file are enabled.
+  DisabledByDefault: true
+  Exclude:
+    - '**/templates/**/*'
+    - '**/vendor/**/*'
+
+# Prefer &&/|| over and/or.
+Style/AndOr:
+  Enabled: true
+
+# Do not use braces for hash literals when they are the last argument of a
+# method call.
+Style/BracesAroundHashParameters:
+  Enabled: true
+
+# Align `when` with `case`.
+Style/CaseIndentation:
+  Enabled: true
+
+# Align comments with method definitions.
+Style/CommentIndentation:
+  Enabled: true
+
+# No extra empty lines.
+Style/EmptyLines:
+  Enabled: true
+
+# In a regular class definition, no empty lines around the body.
+Style/EmptyLinesAroundClassBody:
+  Enabled: true
+
+# In a regular module definition, no empty lines around the body.
+Style/EmptyLinesAroundModuleBody:
+  Enabled: true
+
+# Use Ruby >= 1.9 syntax for hashes. Prefer { a: :b } over { :a => :b }.
+Style/HashSyntax:
+  Enabled: true
+
+# Method definitions after `private` or `protected` isolated calls need one
+# extra level of indentation.
+Style/IndentationConsistency:
+  Enabled: true
+  EnforcedStyle: rails
+
+# Two spaces, no tabs (for indentation).
+Style/IndentationWidth:
+  Enabled: false
+
+# Defining a method with parameters needs parentheses.
+Style/MethodDefParentheses:
+  Enabled: true
+
+# Use `foo {}` not `foo{}`.
+Style/SpaceBeforeBlockBraces:
+  Enabled: true
+
+# Use `foo { bar }` not `foo {bar}`.
+Style/SpaceInsideBlockBraces:
+  Enabled: true
+
+# Use `{ a: 1 }` not `{a:1}`.
+Style/SpaceInsideHashLiteralBraces:
+  Enabled: true
+
+# Check quotes usage according to lint rule below.
+Style/StringLiterals:
+  Enabled: true
+  EnforcedStyle: single_quotes
+
+# Detect hard tabs, no hard tabs.
+Style/Tab:
+  Enabled: true
+
+# Blank lines should not have any spaces.
+Style/TrailingBlankLines:
+  Enabled: true
+
+# No trailing whitespace.
+Style/TrailingWhitespace:
+  Enabled: true
+
+# Use quotes for string literals when they are enough.
+Style/UnneededPercentQ:
+  Enabled: true
+
+# Align `end` with the matching keyword or starting expression except for
+# assignments, where it should be aligned with the LHS.
+Lint/EndAlignment:
+  Enabled: true
+  AlignWith: variable
+
+# Use my_method(my_arg) not my_method( my_arg ) or my_method my_arg.
+Lint/RequireParentheses:
+  Enabled: true

data/.travis.yml

@@ -0,0 +1,9 @@
+sudo: required
+language: ruby
+rvm:
+  - 2.3.1
+before_install: gem install bundler -v 1.13.4
+cache: bundler
+script:
+  - bundle exec rake
+  - bundle exec codeclimate-test-reporter

data/CODE_OF_CONDUCT.md

@@ -0,0 +1,74 @@
+# Contributor Covenant Code of Conduct
+
+## Our Pledge
+
+In the interest of fostering an open and welcoming environment, we as
+contributors and maintainers pledge to making participation in our project and
+our community a harassment-free experience for everyone, regardless of age, body
+size, disability, ethnicity, gender identity and expression, level of experience,
+nationality, personal appearance, race, religion, or sexual identity and
+orientation.
+
+## Our Standards
+
+Examples of behavior that contributes to creating a positive environment
+include:
+
+* Using welcoming and inclusive language
+* Being respectful of differing viewpoints and experiences
+* Gracefully accepting constructive criticism
+* Focusing on what is best for the community
+* Showing empathy towards other community members
+
+Examples of unacceptable behavior by participants include:
+
+* The use of sexualized language or imagery and unwelcome sexual attention or
+advances
+* Trolling, insulting/derogatory comments, and personal or political attacks
+* Public or private harassment
+* Publishing others' private information, such as a physical or electronic
+  address, without explicit permission
+* Other conduct which could reasonably be considered inappropriate in a
+  professional setting
+
+## Our Responsibilities
+
+Project maintainers are responsible for clarifying the standards of acceptable
+behavior and are expected to take appropriate and fair corrective action in
+response to any instances of unacceptable behavior.
+
+Project maintainers have the right and responsibility to remove, edit, or
+reject comments, commits, code, wiki edits, issues, and other contributions
+that are not aligned to this Code of Conduct, or to ban temporarily or
+permanently any contributor for other behaviors that they deem inappropriate,
+threatening, offensive, or harmful.
+
+## Scope
+
+This Code of Conduct applies both within project spaces and in public spaces
+when an individual is representing the project or its community. Examples of
+representing a project or community include using an official project e-mail
+address, posting via an official social media account, or acting as an appointed
+representative at an online or offline event. Representation of a project may be
+further defined and clarified by project maintainers.
+
+## Enforcement
+
+Instances of abusive, harassing, or otherwise unacceptable behavior may be
+reported by contacting the project team at watsy0007@gmail.com. All
+complaints will be reviewed and investigated and will result in a response that
+is deemed necessary and appropriate to the circumstances. The project team is
+obligated to maintain confidentiality with regard to the reporter of an incident.
+Further details of specific enforcement policies may be posted separately.
+
+Project maintainers who do not follow or enforce the Code of Conduct in good
+faith may face temporary or permanent repercussions as determined by other
+members of the project's leadership.
+
+## Attribution
+
+This Code of Conduct is adapted from the [Contributor Covenant][homepage], version 1.4,
+available at [http://contributor-covenant.org/version/1/4][version]
+
+[homepage]: http://contributor-covenant.org
+[version]: http://contributor-covenant.org/version/1/4/

data/Gemfile

@@ -0,0 +1,14 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in crawl_station.gemspec
+gemspec
+
+gem 'thor'
+gem 'celluloid', github: 'celluloid/celluloid', submodules: true
+gem 'wombat', github: 'watsy0007/wombat'
+
+group :test do
+  gem 'byebug'
+  gem 'simplecov'
+  gem 'codeclimate-test-reporter', '1.0.0.pre.rc2'
+end

data/LICENSE.txt

@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright (c) 2016 watsy0007
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

data/README.md

@@ -0,0 +1,43 @@
+# CrawlStation
+
+[![Build Status](https://travis-ci.org/watsy0007/crawl_station.svg?branch=master)](https://travis-ci.org/watsy0007/crawl_station)
+[![Code Climate](https://codeclimate.com/github/watsy0007/crawl_station/badges/gpa.svg)](https://codeclimate.com/github/watsy0007/crawl_station)
+[![Test Coverage](https://codeclimate.com/github/watsy0007/crawl_station/badges/coverage.svg)](https://codeclimate.com/github/watsy0007/crawl_station/coverage)
+[![Issue Count](https://codeclimate.com/github/watsy0007/crawl_station/badges/issue_count.svg)](https://codeclimate.com/github/watsy0007/crawl_station)
+[![Dependency Status](https://gemnasium.com/badges/github.com/watsy0007/crawl_station.svg)](https://gemnasium.com/github.com/watsy0007/crawl_station)
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+```ruby
+gem 'crawl_station'
+```
+
+And then execute:
+
+    $ bundle
+
+Or install it yourself as:
+
+    $ gem install crawl_station
+
+## Usage
+
+[![asciicast](https://asciinema.org/a/90711.png)](https://asciinema.org/a/90711)
+
+## Development
+
+After checking out the repo, run `bin/setup` to install dependencies. Then, run `rake spec` to run the tests. You can also run `bin/console` for an interactive prompt that will allow you to experiment.
+
+To install this gem onto your local machine, run `bundle exec rake install`. To release a new version, update the version number in `version.rb`, and then run `bundle exec rake release`, which will create a git tag for the version, push git commits and tags, and push the `.gem` file to [rubygems.org](https://rubygems.org).
+
+## Contributing
+
+Bug reports and pull requests are welcome on GitHub at https://github.com/[USERNAME]/crawl_station. This project is intended to be a safe, welcoming space for collaboration, and contributors are expected to adhere to the [Contributor Covenant](http://contributor-covenant.org) code of conduct.
+
+
+## License
+
+The gem is available as open source under the terms of the [MIT License](http://opensource.org/licenses/MIT).
+

data/Rakefile

@@ -0,0 +1,6 @@
+require 'bundler/gem_tasks'
+require 'rspec/core/rake_task'
+
+RSpec::Core::RakeTask.new(:spec)
+
+task default: :spec

data/bin/console

@@ -0,0 +1,14 @@
+#!/usr/bin/env ruby
+
+require 'bundler/setup'
+require 'crawl_station'
+
+# You can add fixtures and/or initialization code here to make experimenting
+# with your gem easier. You can also use a different console, if you like.
+
+# (If you use this, don't forget to add pry to your Gemfile!)
+# require 'pry'
+# Pry.start
+
+require 'irb'
+IRB.start

data/bin/setup

@@ -0,0 +1,8 @@
+#!/usr/bin/env bash
+set -euo pipefail
+IFS=$'\n\t'
+set -vx
+
+bundle install
+
+# Do any other automated setup that you need to do here

data/crawl_station.gemspec

@@ -0,0 +1,32 @@
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'crawl_station/version'
+
+Gem::Specification.new do |s|
+  s.name          = 'crawl_station'
+  s.version       = CrawlStation::VERSION
+  s.authors       = ['watsy0007']
+  s.email         = ['watsy0007@gmail.com']
+
+  s.summary       = 'crawl station system'
+  s.description   = 'crawl station system'
+  s.homepage      = 'https://github.io/watsy0007/crawl_sation'
+  s.license       = 'MIT'
+
+  s.required_ruby_version = '>= 2.3.0'
+
+  s.files         = `git ls-files -z`.split("\x0").reject do |f|
+    f.match(%r{^(test|spec|features|lib\/exe)/})
+  end
+  s.bindir        = 'exe'
+  s.executables   = ['station']
+  s.require_paths = ['lib']
+
+  s.add_runtime_dependency 'activesupport', '~> 5.0', '>= 5.0.0'
+  s.add_runtime_dependency 'activerecord', '~> 5.0', '>= 5.0.0'
+  s.add_development_dependency 'bundler', '~> 1.13'
+  s.add_development_dependency 'rake', '~> 11.3'
+  s.add_development_dependency 'rspec', '~> 3.5'
+  s.add_development_dependency 'pry', '~> 0.10'
+end

data/exe/station

@@ -0,0 +1,3 @@
+#!/usr/bin/env ruby
+
+require 'crawl_station/cli'

data/lib/crawl_station/application_record.rb

@@ -0,0 +1,17 @@
+module CrawlStation
+  class ApplicationRecord < ActiveRecord::Base
+    self.abstract_class = true
+
+    def self.table_name_prefix
+      "#{module_name}_"
+    end
+
+    def self.database_config
+      CS::Utils.database_config(module_name)
+    end
+
+    def self.module_name
+      name.to_s.split('::').first.downcase
+    end
+  end
+end

data/lib/crawl_station/cache.rb

@@ -0,0 +1,13 @@
+module CrawlStation
+  module Cache
+    extend self
+    extend Forwardable
+    include Concerns::AdapterConcern
+
+    def adapter=(adapter_name)
+      custom_adapter(adapter_name, CrawlStation::CacheAdapters)
+    end
+
+    def_delegators :@adapter, :[]=, :[], :include?
+  end
+end

data/lib/crawl_station/cache_adapters/abstract_adapter.rb

@@ -0,0 +1,6 @@
+module CrawlStation
+  module CacheAdapters
+    class AbstractAdapter
+    end
+  end
+end

data/lib/crawl_station/cache_adapters/db_adapter.rb

@@ -0,0 +1,6 @@
+module CrawlStation
+  module CacheAdapters
+    class DbAdapter < AbstractAdapter
+    end
+  end
+end

data/lib/crawl_station/cache_adapters/memory_adapter.rb

@@ -0,0 +1,17 @@
+module CrawlStation
+  module CacheAdapters
+    class MemoryAdapter < AbstractAdapter
+      def initialize
+        @cache = {}
+        @cache.extend(MonitorMixin)
+        ['[]=', '[]', 'include?'].each do |method_name|
+          define_singleton_method method_name do |*args|
+            @cache.synchronize do
+              @cache.send(method_name, *args)
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/crawl_station/cli.rb

@@ -0,0 +1,26 @@
+require 'crawl_station/ruby_version_check'
+Signal.trap('INT') { puts; exit(1) }
+require 'crawl_station'
+
+if ARGV.size > 0
+  command = ARGV.shift
+
+  aliases = {
+    'new' => 'create',
+    'g' => 'generate'
+  }
+  command = aliases[command] || command
+  CrawlStation::Command.invoke(command, ARGV)
+  exit(0)
+end
+
+puts "usage:\n"
+%w(
+  create
+  generate
+).each do |command|
+  klass = "#{CrawlStation::Command}::#{command.capitalize}".camelize.constantize
+  klass.tasks.each do |k, v|
+    puts "#{k}: #{v.usage}\n\t#{v.description}\n"
+  end
+end

data/lib/crawl_station/command/create.rb

@@ -0,0 +1,20 @@
+require 'fileutils'
+module CrawlStation
+  module Command
+    class Create < Thor
+      desc 'create station', 'station [create|new] hello'
+      def create(args)
+        dir_root = args.first
+        puts "create project #{dir_root}"
+        template_create_path = "#{CrawlStation::Utils.templates_path}/create"
+        FileUtils.copy_entry template_create_path, dir_root
+        puts "cd #{dir_root}"
+        path = "#{Dir.pwd}/#{dir_root}"
+        Dir.chdir(path)
+        puts 'bundle install'
+        IO.popen('bundle install').each { |line| puts line.chomp }
+        puts 'done'
+      end
+    end
+  end
+end

data/lib/crawl_station/command/generate.rb

@@ -0,0 +1,88 @@
+require 'erb'
+require 'fileutils'
+module CrawlStation
+  module Command
+    class Generate < Thor
+      desc 'generation migration', 'station g migration t66y create_tech_migration'
+      def migration(args)
+        raise "generate #{args} error" if !args.is_a?(Array) || args.size < 2
+        module_name, file_name = args.shift, args.shift
+
+        file_path = dest_path(module_name, file_name)
+        template_path = CS::Utils.template_filepath('generate/migration.erb')
+        render(file_path, template_path, class_name: file_name.camelize)
+        logs "generate migration #{module_name}:#{file_name} done"
+      end
+
+      desc 'create parser module', 'station g module t66y'
+      def new_module(args)
+        raise "create module #{args} error" if !args.is_a?(Array) || args.empty?
+        module_name = args.shift
+        m_path = CS::Utils.module_path(module_name)
+        return logs("#{module_name} module exist!") if File.exist?(m_path)
+        logs "create new module #{module_name}"
+        template_m_path = "#{CS::Utils.templates_path}/generate/module"
+        FileUtils.copy_entry template_m_path, m_path
+        logs "create #{module_name} done"
+      end
+
+      desc 'create parser', 'station g parser t66y tech'
+      def parser(args)
+        raise "generate parser #{args} error" if !args.is_a?(Array) || args.size < 2
+        module_name, parser_name = args.shift, args.shift
+        logs "create #{module_name} parser #{parser_name}"
+        template_parser_path = CS::Utils.template_filepath('generate/parser.erb')
+        template_item_path = CS::Utils.template_filepath('generate/item.erb')
+        opts = {
+          module_class_name: module_name.camelize,
+          class_name: parser_name.camelize
+        }
+        render(parser_path(module_name, parser_name, 'parser'), template_parser_path, opts)
+        render(parser_path(module_name, parser_name, 'item'), template_item_path, opts)
+      end
+
+      desc 'create parsers', 'station g parser t66y tech image japan'
+      def parsers(args)
+        raise "generate parser #{args} error" if !args.is_a?(Array) || args.size < 2
+        module_name = args.shift
+        loop do
+          break if args.empty?
+          parser([module_name, args.shift])
+        end
+      end
+
+      protected
+
+      def render(file_path, template_path, opts = {})
+        File.open(file_path, 'w') do |f|
+          logs "create file #{file_path}"
+          f.write render_context(template_path, opts)
+        end
+      end
+
+      def render_context(path, opts = {})
+        template = IO.read(path)
+        ERB.new(template).result(OpenStruct.new(opts).instance_eval { binding })
+      end
+
+      def logs(msg)
+        CrawlStation.logger.debug msg
+      end
+
+      def dest_path(module_name, file_name)
+        m_path = CrawlStation::Utils.module_path(module_name)
+        raise "module: #{module_name} not exist" unless Dir.exist?(m_path)
+        migrate_path = "#{m_path}/db/migrate"
+        raise "module: migration path #{migrate_path} not exist" unless Dir.exist?(migrate_path)
+        ms = Time.now.to_f.to_s.delete '.'
+        "#{migrate_path}/#{ms}_#{file_name}.rb"
+      end
+
+      def parser_path(module_name, file_name, type = 'parser')
+        m_path = CrawlStation::Utils.module_path(module_name)
+        raise "module: #{module_name} not exist" unless Dir.exist?(m_path)
+        "#{m_path}/#{type}/#{file_name}.rb"
+      end
+    end
+  end
+end

data/lib/crawl_station/command.rb

@@ -0,0 +1,26 @@
+module CrawlStation
+  module Command
+    class CommandError < Exception
+    end
+    extend self
+    ALIASES = { 'module' => 'new_module' }
+
+    def invoke(command, args)
+      klass = "CrawlStation::Command::#{command.capitalize}".camelize.constantize
+      m_name = method_name(command, args)
+      obj = klass.new
+      return obj.send(m_name, args) if obj.respond_to?(m_name)
+      true
+    rescue NameError => e
+      Logger.warn "#{command}: #{args} not exist"
+    rescue Errno::ENOENT, RuntimeError => e
+      Logger.warn "#{e.message}\n#{e.backtrace[0..10].join("\n")}"
+    end
+
+    def method_name(command, args)
+      return 'create' if command.to_s == 'create'
+      name = args.shift
+      ALIASES[name] || name
+    end
+  end
+end

data/lib/crawl_station/concerns/adapter_concern.rb

@@ -0,0 +1,26 @@
+require 'active_support/concern'
+module CrawlStation
+  module Concerns
+    module AdapterConcern
+      extend ActiveSupport::Concern
+
+      class_methods do
+        def adapter
+          return @adapter if @adapter
+          self.adapter = :memory
+          @adapter
+        end
+
+        def custom_adapter(adapter_name, adapter_module)
+          case adapter_name
+          when String, Symbol
+            adapter_full_name = "#{adapter_name}_adapter"
+            @adapter = adapter_module.const_get(adapter_full_name.camelize).new
+          end
+        rescue NameError => e
+          CS.logger.error "Cache missing adapter #{adapter_name}"
+        end
+      end
+    end
+  end
+end

data/lib/crawl_station/configuration.rb

@@ -0,0 +1,7 @@
+module CrawlStation
+  module Configuration
+    extend self
+    attr_accessor :adapter
+    attr_accessor :parsers
+  end
+end