datafile 0.0.1 → 0.1.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 0f6e0ba472e5a12abe56392ae435edbcbe0a441c
-  data.tar.gz: 25e4d1ec36da8cdfe22a5080dc80d5e59c9b85b2
+  metadata.gz: 650b215a771b6d6b984f2a65e187a08803c383ad
+  data.tar.gz: 15512a5c171fea567372ed5983599330ad6312cb
 SHA512:
-  metadata.gz: ba313a8f52adac23f223fb46e797fb7bcb8a30ee20a4e802f165a8a41baa5812fd856e1b14d745147a5791e6e7e512d98706759b40955adc7e37c3c8d884a0fc
-  data.tar.gz: 8fff0d4e06ea2c7017b2288fcc8af541295c35bfcc21f092a43c285ebd5bec56c381080a82bac4ca3a3be3589f1ce1de25a202dd45cf3a91e224d8cd6d36d1a5
+  metadata.gz: ee1d067e74de8bc1d6ba85cb1cc235b30954d7b0e88254d487ab40a4050143439aa0d994eb34e5801cbb61a1a8c2a28da974878d43a56615118d2bae1a31fd80
+  data.tar.gz: 1c019e305e059a04d3cd7a25909d5616e9d13c5bdbb8a4c360eb042f04484476655e701411c2ecc24e255647c8567171607c58532bb4dd684d90a941cc38c1b1

data/Manifest.txt

@@ -3,4 +3,11 @@ Manifest.txt
 README.md
 Rakefile
 lib/datafile.rb
+lib/datafile/builder.rb
+lib/datafile/datafile.rb
+lib/datafile/datasets/dataset.rb
+lib/datafile/datasets/football.rb
+lib/datafile/datasets/world.rb
 lib/datafile/version.rb
+test/helper.rb
+test/test_builder.rb

data/Rakefile

@@ -18,7 +18,9 @@ Hoe.spec 'datafile' do
   self.history_file = 'HISTORY.md'
 
   self.extra_deps = [
-    ['logutils' ]
+    ['logutils'],
+    ['textutils'],
+    ['fetcher'],
   ]
 
   self.licenses = ['Public Domain']
@@ -26,6 +28,4 @@ Hoe.spec 'datafile' do
   self.spec_extras = {
    :required_ruby_version => '>= 1.9.2'
   }
-
-
 end

data/lib/datafile.rb

@@ -1,4 +1,4 @@
-
+# encoding: utf-8
 
 require 'logutils'
 
@@ -7,19 +7,11 @@ require 'logutils'
 
 require 'datafile/version'  # let it always go first
 
-
-
-module Datafile
-
-  def self.banner
-    "datafile/#{VERSION} on Ruby #{RUBY_VERSION} (#{RUBY_RELEASE_DATE}) [#{RUBY_PLATFORM}]"
-  end
-
-  def self.root
-    "#{File.expand_path( File.dirname(File.dirname(__FILE__)) )}"
-  end  
-
-end  # module Datafile
+require 'datafile/datasets/dataset'
+require 'datafile/datasets/football'
+require 'datafile/datasets/world'
+require 'datafile/datafile'
+require 'datafile/builder'
 
 
 

data/lib/datafile/builder.rb

@@ -0,0 +1,38 @@
+# encoding: utf-8
+
+module Datafile
+
+class Builder
+
+  include LogUtils::Logging
+
+  attr_reader :datafile
+
+  def initialize()
+    @datafile = Datafile.new
+  end
+
+  def self.load_file( path )
+    code = File.read_utf8( path )
+    self.load( code )
+  end
+
+  def self.load( code )
+    builder = Builder.new
+    builder.instance_eval( code )
+    builder
+  end
+
+
+  def football( name, opts={} )
+    logger.info( "[builder] add football-dataset '#{name}'" )
+    @datafile.datasets << FootballDataset.new( name, opts )
+  end
+
+  def world( name, opts={} )
+    logger.info( "[builder] add world-dataset '#{name}'" )
+    @datafile.datasets << WorldDataset.new( name, opts )
+  end
+
+end # class Builder
+end # module Datafile

data/lib/datafile/datafile.rb

@@ -0,0 +1,39 @@
+# encoding: utf-8
+
+module Datafile
+
+class Datafile
+
+  include LogUtils::Logging
+
+  attr_reader :datasets
+
+  def initialize()
+    @datasets = []
+  end
+
+
+  def run()
+    logger.info( "[datafile] begin - run" )
+    download()   # step 1 - download zips for datasets
+    read()       # step 2 - read in datasets from zips
+    logger.info( "[datafile] end - run" )
+  end
+
+  def download()
+    logger.info( "[datafile] dowload" )
+    @datasets.each do |dataset|
+      dataset.download()
+    end
+  end
+
+  def read()
+    logger.info( "[datafile] read" )
+    @datasets.each do |dataset|
+      dataset.read()
+    end
+  end
+
+end # class Datafile
+end # module Datafile
+

data/lib/datafile/datasets/dataset.rb

@@ -0,0 +1,63 @@
+# encoding: utf-8
+
+module Datafile
+
+class Dataset
+  include LogUtils::Logging
+
+  def initialize( name, opts={} )
+    @name = name
+    @opts = opts
+  end
+
+  def setup()
+    value = @opts[:setup] || 'all'
+    "setups/#{value}"
+  end
+
+  def remote_zip_url()  # remote zip url
+    "https://github.com/#{@name}/archive/master.zip"
+  end
+
+  def local_zip_name()
+    ### note: replace / in name w/ --I--
+    ##  e.g. flatten the filename, that is, do NOT include any folders
+    @name.gsub('/', '--I--')   # note: will NOT include/return .zip extension
+  end
+
+  def local_zip_root()
+    "./tmp"
+  end
+
+  def local_zip_path()  # local zip path
+    "#{local_zip_root}/#{local_zip_name}.zip"
+  end
+
+
+  def download()
+    logger.info( "download dataset '#{@name}'" )
+    logger.info( "   from '#{remote_zip_url}'" )
+    logger.info( "   to '#{local_zip_path}'..." )
+
+    download_blob( remote_zip_url, local_zip_path )
+  end
+
+private
+  ####
+  # download tasks for zips
+  def download_blob( url, dest )
+    logger.info "downloading #{url} to #{dest}..."
+
+    ## make sure dest path exists
+    dest_p = File.dirname( dest )
+    FileUtils.mkdir_p( dest_p )  unless File.exists?( dest_p )   ## use Dir.exists?? why? why not??
+
+    worker = Fetcher::Worker.new
+    worker.copy( url, dest )
+    ## print some file stats
+    logger.debug "  size: #{File.size(dest)} bytes"
+  end
+
+end  # class Dataset
+
+end # module Datafile

data/lib/datafile/datasets/football.rb

@@ -0,0 +1,18 @@
+# encoding: utf-8
+
+module Datafile
+
+class FootballDataset < Dataset
+
+  def initialize( name, opts={} )
+    super( name, opts )
+  end
+
+  def read()
+    logger.info( "read football-dataset '#{@name}', '#{setup}'" )
+
+    SportDb.read_setup_from_zip( local_zip_name(), setup(), local_zip_root() )
+  end
+end  # class FootballDataset
+
+end # module Datafile

data/lib/datafile/datasets/world.rb

@@ -0,0 +1,19 @@
+# encoding: utf-8
+
+module Datafile
+
+class WorldDataset < Dataset
+
+  def initialize( name, opts={} )
+    super( name, opts )
+  end
+
+  def read()
+    logger.info( "read world-dataset '#{@name}', '#{setup}'" )
+
+    WorldDb.read_setup_from_zip( local_zip_name(), setup(), local_zip_root(), { skip_tags: true } )
+  end
+end  # class WorldDataset
+
+end # module Datafile
+

data/lib/datafile/version.rb

@@ -1,7 +1,23 @@
-
+# encoding: utf-8
 
 module Datafile
-   VERSION = '0.0.1'
-end
 
+  MAJOR = 0    ## todo: namespace inside version or something - why? why not??
+  MINOR = 1
+  PATCH = 0
+  VERSION = [MAJOR,MINOR,PATCH].join('.')
+
+  def self.version
+    VERSION
+  end
+
+  def self.banner
+    "datafile/#{VERSION} on Ruby #{RUBY_VERSION} (#{RUBY_RELEASE_DATE}) [#{RUBY_PLATFORM}]"
+  end
+
+  def self.root
+    "#{File.expand_path( File.dirname(File.dirname(File.dirname(__FILE__))) )}"
+  end  
+
+end  # module Datafile
 

data/test/helper.rb

@@ -0,0 +1,12 @@
+
+## minitest setup
+require 'minitest/autorun'
+
+## deps
+require 'worlddb'
+require 'sportdb'   # note: will include worlddb
+
+
+## our own code
+require 'datafile'
+

data/test/test_builder.rb

@@ -0,0 +1,39 @@
+# encoding: utf-8
+
+###
+#  to run use
+#     ruby -I ./lib -I ./test test/test_builder.rb
+
+
+require 'helper'
+
+class TestBuilder < MiniTest::Test
+
+  def test_builder
+    code =<<EOS
+## comments
+
+  world  'openmundi/world.db', setup: 'countries'
+
+  football 'openfootball/national-teams'      ## NOTE: default is setup: 'all'
+  
+  ### todo/fix: download archive only once(!!) even if included more than once
+  ## football 'openfootball/world-cup', setup: '2014_quali'
+
+  football 'openfootball/world-cup', setup: '2014'
+
+## more comments
+
+EOS
+
+    builder = Datafile::Builder.load( code )
+    
+    datafile = builder.datafile
+    ## datafile.run
+    ## datafile.download
+    ## datafile.read
+
+    assert true  # if we get here - test success
+  end
+
+end # class TestBuilder

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: datafile
 version: !ruby/object:Gem::Version
-  version: 0.0.1
+  version: 0.1.0
 platform: ruby
 authors:
 - Gerald Bauer
@@ -24,6 +24,34 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: textutils
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: fetcher
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: rdoc
   requirement: !ruby/object:Gem::Requirement
@@ -61,12 +89,20 @@ extra_rdoc_files:
 - Manifest.txt
 - README.md
 files:
+- ".gemtest"
 - HISTORY.md
 - Manifest.txt
 - README.md
 - Rakefile
 - lib/datafile.rb
+- lib/datafile/builder.rb
+- lib/datafile/datafile.rb
+- lib/datafile/datasets/dataset.rb
+- lib/datafile/datasets/football.rb
+- lib/datafile/datasets/world.rb
 - lib/datafile/version.rb
+- test/helper.rb
+- test/test_builder.rb
 homepage: https://github.com/rubylibs/datafile
 licenses:
 - Public Domain
@@ -93,4 +129,5 @@ rubygems_version: 2.4.2
 signing_key: 
 specification_version: 4
 summary: datafile - builder for downloading n reading datasets
-test_files: []
+test_files:
+- test/test_builder.rb