tottori-opendata-catalog 0.0.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 22a6e0f965e1ee16d9b023d34ac2054505c68e19
+  data.tar.gz: 90e44e14b33eefa0709e9c85ccf4cdab923e74ed
+SHA512:
+  metadata.gz: 0a9b699b1b35cd0c81d3f55b5d4d757a95aa6d5a79a884416f641a97bd897915633c0e770e272269e47ce2560c900ff3571b6d57dd4e60d3727c0085791c295c
+  data.tar.gz: 0b8e2da856bf8a45e8cb2a6ae218f3fe047cc844c6c6f2b9420bbc4a7c11f6048041cfd04a99c72c669f46e21dcfd4cb789e54ec2a912e2e54ea37f5d0984777

data/.gitignore

@@ -0,0 +1,22 @@
+*.gem
+*.rbc
+.bundle
+.config
+.yardoc
+Gemfile.lock
+InstalledFiles
+_yardoc
+coverage
+doc/
+lib/bundler/man
+pkg
+rdoc
+spec/reports
+test/tmp
+test/version_tmp
+tmp
+*.bundle
+*.so
+*.o
+*.a
+mkmf.log

data/.rspec

@@ -0,0 +1,2 @@
+--format documentation
+--color

data/.travis.yml

@@ -0,0 +1,3 @@
+language: ruby
+rvm:
+  - 2.1.1

data/Gemfile

@@ -0,0 +1,4 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in tottori-opendata-catalog.gemspec
+gemspec

data/LICENSE.txt

@@ -0,0 +1,22 @@
+Copyright (c) 2014 Masayuki Higashino
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,39 @@
+[![Version](https://img.shields.io/gem/v/cardamon.svg)](https://rubygems.org/gems/tottori-opendata-catalog)
+
+## TottoriOpenDataCatalog
+
+This gem provides an API and a CLI of Open Data Catalog of Tottori Prefecture http://db.pref.tottori.jp/opendataResearch.nsf to generate a machine-readable catalog.
+
+### Installation
+
+    $ gem install cardamon
+
+### Synopsis
+
+#### API
+
+``` ruby
+require 'tottori-opendata-catalog'
+# TottoriOpenDataCatalog.get returns a nested ruby's object.
+puts TottoriOpenDataCatalog.get
+```
+
+``` ruby
+require 'tottori-opendata-catalog'
+# without caching using a local strage (/tmp).
+puts TottoriOpenDataCatalog.get(cache:false)
+```
+
+#### CLI
+
+``` sh
+$ tottori-opendata-catalog csv > catalog.csv
+```
+
+``` sh
+$ tottori-opendata-catalog json > catalog.json
+```
+
+``` sh
+$ tottori-opendata-catalog json --no-cache > catalog.json
+```

data/Rakefile

@@ -0,0 +1,7 @@
+require "bundler/gem_tasks"
+require "rspec/core/rake_task"
+
+RSpec::Core::RakeTask.new(:spec)
+
+task :default => :spec
+

data/bin/tottori-opendata-catalog

@@ -0,0 +1,18 @@
+#!/usr/bin/env ruby
+
+require 'tottori-opendata-catalog'
+require 'thor'
+
+class CLI < Thor
+  option :cache, {type: :boolean, default:true}
+  desc 'csv', 'print a csv file'
+  def csv
+    puts TottoriOpenDataCatalog::Command.to_csv(cache:options[:cache])
+  end
+  desc 'json', 'print a json file'
+  def json
+    puts TottoriOpenDataCatalog::Command.to_json(cache:options[:cache])
+  end
+end
+
+CLI.start(ARGV)

data/lib/tottori-opendata-catalog.rb

@@ -0,0 +1,13 @@
+require 'tottori-opendata-catalog/version'
+require 'tottori-opendata-catalog/parser'
+require 'tottori-opendata-catalog/net'
+require 'tottori-opendata-catalog/proxy'
+require 'tottori-opendata-catalog/command'
+
+module TottoriOpenDataCatalog
+
+  def self.get(cache:true)
+    TottoriOpenDataCatalog::Proxy.get(cache:cache)
+  end
+
+end

data/lib/tottori-opendata-catalog/command.rb

@@ -0,0 +1,57 @@
+require 'tottori-opendata-catalog/proxy'
+
+module TottoriOpenDataCatalog
+
+  module Command
+
+    class << self
+
+      def to_json(cache:true)
+        require 'json'
+        JSON.pretty_generate(TottoriOpenDataCatalog::Proxy.get(cache:cache))
+      end
+
+      def to_csv(cache:true)
+        require 'csv'
+        CSV(csv = '') do |line|
+          data = TottoriOpenDataCatalog::Proxy.get(cache:cache)
+          line << %w{
+            category_name
+            resource_name formats department division tags
+            redistribution_allowed commercial_use_allowed
+            provider tel description comment
+            year month day
+            repeat_rule_frequency repeat_rule_interval repeat_rule_description
+          }
+          data[:categories].each do |category|
+            category[:resources].each do |resource|
+              row = []
+              row << category[:name]
+              row << resource[:name]
+              row << resource[:formats].join(' ')
+              row << resource[:department]
+              row << resource[:division]
+              row << resource[:tags].join(' ')
+              row << resource[:redistribution_allowed]
+              row << resource[:commercial_use_allowed]
+              row << resource[:provider]
+              row << resource[:tel]
+              row << resource[:description]
+              row << resource[:comment]
+              row << resource[:year]
+              row << resource[:month]
+              row << resource[:day]
+              row << resource[:repeat_rule][:frequency]
+              row << resource[:repeat_rule][:interval]
+              row << resource[:repeat_rule][:description]
+              line << row
+            end
+          end
+        end
+        return csv
+      end
+    end
+
+  end
+
+end

data/lib/tottori-opendata-catalog/net.rb

@@ -0,0 +1,32 @@
+require 'open-uri'
+require 'digest/sha1'
+require 'tmpdir'
+
+module TottoriOpenDataCatalog
+
+  module Net
+
+    class << self
+
+      def get(url, cache:true)
+        return open(url, &:read) unless cache
+        path = File.join(Dir.tmpdir, Digest::SHA1.hexdigest(url))
+        @memo ||= {}
+        if @memo.include?(path)
+          @memo[path]
+        else
+          if File.exists?(path)
+            @memo[path] ||= open(path, &:read)
+          else
+            content = open(url, &:read)
+            open(path, 'w'){ |io| io.write(content) }
+            @memo[path] ||= content
+          end
+        end
+      end
+
+    end
+
+  end
+
+end

data/lib/tottori-opendata-catalog/parser.rb

@@ -0,0 +1,136 @@
+require 'nokogiri'
+
+module TottoriOpenDataCatalog
+
+  module Parser
+
+    class << self
+
+      def parse_index(string)
+        doc = Nokogiri::HTML(string, nil, 'Shift_JIS')
+        # collect
+        name = doc.xpath('//title').text.strip
+        anchors = doc.xpath('//a')
+        categories = anchors.select{ |a|
+          a[:href].include?('forweb_bunrui')
+        }.map{ |a|
+          {name:a.text.strip, link:a[:href].strip}
+        }
+        items = {
+          name:name,
+          categories:categories,
+        }
+        return items
+      end
+
+      def parse_list(string)
+        doc = Nokogiri::HTML(string, nil, 'Shift_JIS')
+        # collect
+        items = doc.xpath('//table[@id="contentslist"]/tr[position() > 1]').map{ |tr|
+          tds = tr.children
+          item = {
+            name:tds[0].text.strip,
+            link:tds[0].children[0][:href],
+            formats:tds[1].text.strip,
+            url:tds[2].children[0][:href],
+            department:tds[3].text.strip,
+            division:tds[4].text.strip,
+          }
+        }
+        # trim
+        items.each{ |item|
+          if item[:formats]
+            item[:formats].upcase!
+            item[:formats] = item[:formats].split
+          end
+        }
+        # HACK change format from Notes to PDF.
+        target = items.select{ |item|
+          item[:link] == 'list1_forweb/10EE759AAD20B54749257C68000A2845?OpenDocument'
+        }.first
+        target[:formats] = ['PDF'] if target
+        return items
+      end
+
+      def parse_record(string)
+        doc = Nokogiri::HTML(string, nil, 'Shift_JIS')
+        # collect
+        item = doc.xpath('//form/div[@id="all"]').children.map{ |e|
+          case e.text.strip
+          when /データ年次.*?：(.*)$/
+            {updated_at:$1.strip}
+          when /情報.*?：(.*)/
+            {tags:$1.strip}
+          when /再配布可否.*?：(.*)$/
+            {redistribution_allowed:$1.strip}
+          when /商用利用可否.*?：(.*)$/
+            {commercial_use_allowed:$1.strip}
+          when /テータ提供.*?：(.*)$/
+            {provider:$1.strip}
+          when /問い合わせ先電話番号.*?：(.*)$/
+            {tel:$1.strip}
+          when /内容.*?：(.*)$/
+            {description:$1.strip}
+          when /コメント.*?：(.*)$/
+            {comment:$1.strip}
+          end
+        }.compact.inject(&:merge)
+        # trim
+        if item[:redistribution_allowed] == '○'
+          item[:redistribution_allowed] = true
+        end
+        if item[:commercial_use_allowed] == '○'
+          item[:commercial_use_allowed] = true
+        end
+        updated_at = item.delete(:updated_at)
+        if updated_at
+          updated_at.gsub!('　', ' ')
+          updated_at.tr!('０-９', '0-9')
+        end
+        date, repeat_rule_description = updated_at.split
+        year, month, day = date.split('.')
+        item[:year]  = (year  ? year.to_i  : nil)
+        item[:month] = (month ? month.to_i : nil)
+        item[:day]   = (day   ? day.to_i   : nil)
+        item[:repeat_rule] = {}
+        item[:repeat_rule].merge!(
+          case repeat_rule_description
+          when /毎年度/
+            {frequency:'fiscal_yearly', interval:nil}
+          when /毎年/
+            {frequency:'yearly', interval:nil}
+          when /毎月/
+            {frequency:'monthly', interval:nil}
+          when /毎週/
+            {frequency:'weekly', interval:nil}
+          when /(\d)年度毎/
+            {frequency:'fiscal_yearly', interval:$1.to_i}
+          when /(\d)年毎/
+            {frequency:'yearly', interval:$1.to_i}
+          when /(\d)月毎/, /(\d)ヶ月毎/
+            {frequency:'monthly', interval:$1.to_i}
+          when /(\d)週毎/
+            {frequency:'weekly', interval:$1.to_i}
+          when /随時/
+            {frequency:'as_needed', interval:nil}
+          when /不定期/
+            {frequency:'unscheduled', interval:nil}
+          else
+            {frequency:nil, interval:nil}
+        end)
+        item[:repeat_rule].merge!(description:repeat_rule_description)
+        # 繰り返しルールの説明が有るのに周期が無い場合は例外をスロー
+        if !item[:repeat_rule][:description].nil? && item[:repeat_rule][:frequency].nil?
+          raise
+        end
+        if item.include?(:tags)
+          item[:tags] = item[:tags].split
+        end
+        return item
+      end
+
+    end
+
+  end
+
+end

data/lib/tottori-opendata-catalog/proxy.rb

@@ -0,0 +1,28 @@
+
+module TottoriOpenDataCatalog
+
+  module Proxy
+
+    INDEX_URL = 'http://db.pref.tottori.jp/opendataResearch.nsf/index.html'
+
+    class << self
+
+      def get(cache:true)
+        index = Parser.parse_index(Net.get(INDEX_URL, cache:cache))
+        index[:categories].each do |category|
+          category[:link] = File.join(File.dirname(INDEX_URL), category[:link])
+          category[:resources] ||= []
+          Parser.parse_list(Net.get(category[:link], cache:cache)).each do |resource|
+            resource[:link] = File.join(File.dirname(INDEX_URL), resource[:link])
+            resource.merge!(Parser.parse_record(Net.get(resource[:link], cache:cache)))
+            category[:resources] << resource
+          end
+        end
+        return index
+      end
+
+    end
+
+  end
+
+end

data/lib/tottori-opendata-catalog/version.rb

@@ -0,0 +1,3 @@
+module TottoriOpenDataCatalog
+  VERSION = '0.0.1'
+end

data/spec/spec_helper.rb

@@ -0,0 +1,3 @@
+$LOAD_PATH.unshift File.expand_path('../../lib', __FILE__)
+require 'tottori-opendata-catalog'
+require 'facets/kernel/blank'

data/spec/tottori-opendata-catalog/command_spec.rb

@@ -0,0 +1,25 @@
+require 'spec_helper'
+
+describe TottoriOpenDataCatalog::Command do
+
+  describe '#to_csv' do
+
+    it 'does not raise errors.' do
+      expect do
+        TottoriOpenDataCatalog::Command.to_csv
+      end.not_to raise_error
+    end
+
+  end
+
+  describe '#to_json' do
+
+    it 'does not raise errors.' do
+      expect do
+        TottoriOpenDataCatalog::Command.to_json
+      end.not_to raise_error
+    end
+
+  end
+
+end

data/spec/tottori-opendata-catalog/parser_spec.rb

@@ -0,0 +1,82 @@
+require 'spec_helper'
+include TottoriOpenDataCatalog
+include TottoriOpenDataCatalog::Proxy
+
+describe TottoriOpenDataCatalog::Parser do
+
+  def parse_index
+    @index ||= Parser.parse_index(Net.get(INDEX_URL))
+  end
+
+  def parse_list(path)
+    url = File.join(File.dirname(INDEX_URL), path)
+    @lists ||= {}
+    @lists[url] ||= Parser.parse_list(Net.get(url))
+  end
+
+  def parse_record(path)
+    url = File.join(File.dirname(INDEX_URL), path)
+    @records ||= {}
+    @records[url] ||= Parser.parse_record(Net.get(url))
+  end
+
+  describe '#parse_index' do
+
+    it 'does not raise errors.' do
+      expect do
+        parse_index
+      end.not_to raise_error
+    end
+
+    it 'returns valid data.' do
+      index = parse_index
+      expect(index[:name].blank?).to be false
+      expect(index[:categories].blank?).to be false
+      index[:categories].each do |category|
+        expect(category[:name].blank?).to be false
+        expect(category[:link].blank?).to be false
+      end
+    end
+
+  end
+
+  describe '#parse_list' do
+
+    it 'does not raise errors.' do
+      parse_index[:categories].each do |list|
+        expect do
+          parse_list(list[:link])
+        end.not_to raise_error
+      end
+    end
+
+    it 'returns valid data.' do
+      parse_index[:categories].each do |list|
+        parse_list(list[:link]).each do |table|
+          expect(table[:name].blank?).to be false
+          expect(table[:link].blank?).to be false
+        end
+      end
+    end
+
+  end
+
+  describe '#parse_record' do
+
+    it 'does not raise errors.' do
+      parse_index[:categories].each do |list|
+        parse_list(list[:link]).each do |record|
+          expect do
+            parse_record(record[:link])
+          end.not_to raise_error
+        end
+      end
+    end
+
+    it 'returns valid data.' do
+      # TODO ;)
+    end
+
+  end
+
+end

data/spec/tottori-opendata-catalog/proxy_spec.rb

@@ -0,0 +1,15 @@
+require 'spec_helper'
+
+describe TottoriOpenDataCatalog::Proxy do
+
+  describe '#get' do
+
+    it 'does not raise errors.' do
+      expect do
+        TottoriOpenDataCatalog::Proxy.get
+      end.not_to raise_error
+    end
+
+  end
+
+end

data/tottori-opendata-catalog.gemspec

@@ -0,0 +1,26 @@
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'tottori-opendata-catalog/version'
+
+Gem::Specification.new do |spec|
+  spec.name          = "tottori-opendata-catalog"
+  spec.version       = TottoriOpenDataCatalog::VERSION
+  spec.authors       = ["Masayuki Higashino"]
+  spec.email         = ["mh.on.web@gmail.com"]
+  spec.summary       = %q{An API and a CLI for Open Data Catalog of Tottori Prefecture.}
+  spec.description   = %q{An API and a CLI for Open Data Catalog of Tottori Prefecture.}
+  spec.homepage      = "https://github.com/mh61503891/tottori-opendata-catalog"
+  spec.license       = "MIT"
+
+  spec.files         = `git ls-files -z`.split("\x0")
+  spec.executables   = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
+  spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
+  spec.require_paths = ["lib"]
+
+  spec.add_dependency "thor", "~> 0.19.1"
+  spec.add_dependency "nokogiri", "~> 1.6.1"
+  spec.add_development_dependency "bundler", "~> 1.6"
+  spec.add_development_dependency "rake"
+  spec.add_development_dependency "rspec"
+end

metadata

@@ -0,0 +1,139 @@
+--- !ruby/object:Gem::Specification
+name: tottori-opendata-catalog
+version: !ruby/object:Gem::Version
+  version: 0.0.1
+platform: ruby
+authors:
+- Masayuki Higashino
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2014-04-06 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: thor
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 0.19.1
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 0.19.1
+- !ruby/object:Gem::Dependency
+  name: nokogiri
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 1.6.1
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 1.6.1
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '1.6'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '1.6'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+description: An API and a CLI for Open Data Catalog of Tottori Prefecture.
+email:
+- mh.on.web@gmail.com
+executables:
+- tottori-opendata-catalog
+extensions: []
+extra_rdoc_files: []
+files:
+- .gitignore
+- .rspec
+- .travis.yml
+- Gemfile
+- LICENSE.txt
+- README.md
+- Rakefile
+- bin/tottori-opendata-catalog
+- lib/tottori-opendata-catalog.rb
+- lib/tottori-opendata-catalog/command.rb
+- lib/tottori-opendata-catalog/net.rb
+- lib/tottori-opendata-catalog/parser.rb
+- lib/tottori-opendata-catalog/proxy.rb
+- lib/tottori-opendata-catalog/version.rb
+- spec/spec_helper.rb
+- spec/tottori-opendata-catalog/command_spec.rb
+- spec/tottori-opendata-catalog/parser_spec.rb
+- spec/tottori-opendata-catalog/proxy_spec.rb
+- tottori-opendata-catalog.gemspec
+homepage: https://github.com/mh61503891/tottori-opendata-catalog
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.0.3
+signing_key: 
+specification_version: 4
+summary: An API and a CLI for Open Data Catalog of Tottori Prefecture.
+test_files:
+- spec/spec_helper.rb
+- spec/tottori-opendata-catalog/command_spec.rb
+- spec/tottori-opendata-catalog/parser_spec.rb
+- spec/tottori-opendata-catalog/proxy_spec.rb
+has_rdoc: