csvpack 0.1.0 → 0.2.0

data/Rakefile

@@ -1,32 +1,32 @@
-require 'hoe'
-require './lib/csvpack/version.rb'
-
-Hoe.spec 'csvpack' do
-
-  self.version = CsvPack::VERSION
-
-  self.summary = 'csvpack - work with tabular data packages using comma-separated values (CSV) datafiles in text with datapackage.json; download, read into and query comma-separated values (CSV) datafiles with your SQL database (e.g. SQLite, PostgreSQL, ...) of choice and much more'
-  self.description = summary
-
-  self.urls    = ['https://github.com/csv11/csvpack']
-
-  self.author  = 'Gerald Bauer'
-  self.email   = 'ruby-talk@ruby-lang.org'
-
-  # switch extension to .markdown for gihub formatting
-  self.readme_file  = 'README.md'
-  self.history_file = 'HISTORY.md'
-
-  self.extra_deps = [
-    ['logutils',     '>=0.6.1'],
-    ['fetcher',      '>=0.4.5'],
-    ['activerecord', '>=5.0.0'],
-  ]
-
-  self.licenses = ['Public Domain']
-
-  self.spec_extras = {
-    required_ruby_version: '>= 2.2.2'
-  }
-
-end
+require 'hoe'
+require './lib/csvpack/version.rb'
+
+Hoe.spec 'csvpack' do
+
+  self.version = CsvPack::VERSION
+
+  self.summary = 'csvpack - work with tabular data packages using comma-separated values (CSV) datafiles in text with datapackage.json; download, read into and query comma-separated values (CSV) datafiles with your SQL database (e.g. SQLite, PostgreSQL, ...) of choice and much more'
+  self.description = summary
+
+  self.urls    = ['https://github.com/csv11/csvpack']
+
+  self.author  = 'Gerald Bauer'
+  self.email   = 'ruby-talk@ruby-lang.org'
+
+  # switch extension to .markdown for gihub formatting
+  self.readme_file  = 'README.md'
+  self.history_file = 'HISTORY.md'
+
+  self.extra_deps = [
+    ['logutils',     '>=0.6.1'],
+    ['fetcher',      '>=0.4.5'],
+    ['activerecord', '>=5.0.0'],
+  ]
+
+  self.licenses = ['Public Domain']
+
+  self.spec_extras = {
+    required_ruby_version: '>= 2.2.2'
+  }
+
+end

data/lib/csvpack.rb

@@ -1,52 +1,52 @@
-# encoding: utf-8
-
-
-require 'pp'
-require 'forwardable'
-
-### csv
-require 'csv'
-require 'json'
-require 'fileutils'
-
-
-### downloader
-require 'fetcher'
-
-### activerecord w/ sqlite3
-##  require 'active_support/all'    ## needed for String#binary? method
-require 'active_record'
-
-
-
-# our own code
-
-require 'csvpack/version'      ## let version always go first
-require 'csvpack/pack'
-require 'csvpack/downloader'
-
-module CsvPack
-
-  def self.import( *args )
-    ## step 1: download
-    dl = Downloader.new
-    args.each do |arg|
-      dl.fetch( arg )
-    end
-
-    ## step 2: up 'n' import
-    args.each do |arg|
-      pack = Pack.new( "./pack/#{arg}/datapackage.json" )
-      pack.tables.each do |table|
-        table.up!
-        table.import!
-      end
-    end
-  end
-
-end # module CsvPack
-
-
-
-# say hello
-puts CsvPack.banner    if defined?($RUBYLIBS_DEBUG) && $RUBYLIBS_DEBUG
+# encoding: utf-8
+
+
+require 'pp'
+require 'forwardable'
+
+### csv
+require 'csv'
+require 'json'
+require 'fileutils'
+
+
+### downloader
+require 'fetcher'
+
+### activerecord w/ sqlite3
+##  require 'active_support/all'    ## needed for String#binary? method
+require 'active_record'
+
+
+
+# our own code
+
+require 'csvpack/version'      ## let version always go first
+require 'csvpack/pack'
+require 'csvpack/downloader'
+
+module CsvPack
+
+  def self.import( *args )
+    ## step 1: download
+    dl = Downloader.new
+    args.each do |arg|
+      dl.fetch( arg )
+    end
+
+    ## step 2: up 'n' import
+    args.each do |arg|
+      pack = Pack.new( "./pack/#{arg}/datapackage.json" )
+      pack.tables.each do |table|
+        table.up!
+        table.import!
+      end
+    end
+  end
+
+end # module CsvPack
+
+
+
+# say hello
+puts CsvPack.banner    if defined?($RUBYLIBS_DEBUG) && $RUBYLIBS_DEBUG

data/lib/csvpack/downloader.rb

@@ -1,62 +1,72 @@
-# encoding: utf-8
-
-module CsvPack
-
-class Downloader
-
-  def initialize( cache_dir='./pack' )
-    @cache_dir = cache_dir   # todo: check if folder exists now (or on demand)?
-    @worker = Fetcher::Worker.new
-  end
-
-  SHORTCUTS = {
-    ## to be done
-  }
-
-  def fetch( name_or_shortcut_or_url )   ## todo/check: use (re)name to get/update/etc. why? why not??
-
-    name = name_or_shortcut_or_url
-
-    ##
-    ## e.g. try
-    ##   country-list
-    ##
-
-    ## url_base = "http://data.okfn.org/data/core/#{name}"
-    url_base = "https://datahub.io/core/#{name}"
-    url = "#{url_base}/datapackage.json"
-
-    dest_dir = "#{@cache_dir}/#{name}"
-    FileUtils.mkdir_p( dest_dir )
-
-    pack_path = "#{dest_dir}/datapackage.json"
-    @worker.copy( url, pack_path )
-
-    h = JSON.parse( File.read( pack_path ) )
-    pp h
-
-    ## copy resources (tables)
-    h['resources'].each do |r|
-      puts "== resource:"
-      pp r
-
-      res_url       = r['url']
-
-      res_name          = r['name']
-      res_relative_path = r['path']
-      if res_relative_path.nil?
-        res_relative_path = "#{res_name}.csv"
-      end
-
-      res_path = "#{dest_dir}/#{res_relative_path}"
-      puts "[debug] res_path: >#{res_path}<"
-      res_dir   = File.dirname( res_path )
-      FileUtils.mkdir_p( res_dir )
-
-      @worker.copy( res_url, res_path )
-    end
-  end
-
-end # class Downloader
-
-end # module CsvPack
+# encoding: utf-8
+
+module CsvPack
+
+class Downloader
+
+  def initialize( cache_dir='./pack' )
+    @cache_dir = cache_dir   # todo: check if folder exists now (or on demand)?
+    @worker = Fetcher::Worker.new
+  end
+
+  SHORTCUTS = {
+    ## to be done
+  }
+
+  def fetch( name_or_shortcut_or_url )   ## todo/check: use (re)name to get/update/etc. why? why not??
+
+    name = name_or_shortcut_or_url
+
+    ##
+    ## e.g. try
+    ##   country-list
+    ##
+
+    ## url_base = "http://data.okfn.org/data/core/#{name}"
+    ## url_base = "https://datahub.io/core/#{name}"
+
+    ## or use "https://github.com/datasets/#{name}/raw/master"
+    url_base = "https://raw.githubusercontent.com/datasets/#{name}/master"
+
+
+    url = "#{url_base}/datapackage.json"
+
+    dest_dir = "#{@cache_dir}/#{name}"
+    FileUtils.mkdir_p( dest_dir )
+
+    pack_path = "#{dest_dir}/datapackage.json"   ## todo/fix: rename to meta_path - why? why not?
+    @worker.copy( url, pack_path )
+
+    h = Meta.load_file( pack_path )
+    pp h
+
+    ## copy resources (tables)
+    h.resources.each do |r|
+      puts "== resource:"
+      pp r
+
+      res_name          = r['name']
+      res_relative_path = r['path']   ## fix/todo: might no contain the url - is now res_url_or_relative_path !!!!!
+      if res_relative_path.nil?
+        res_relative_path = "#{res_name}.csv"
+      end
+
+      res_url       = r['url']   ## check - old package format - url NO longer used!!!!
+      if res_url.nil?
+         ## build url
+         res_url = "#{url_base}/#{res_relative_path}"
+      end
+
+      ## todo/fix: rename - use just res_path - why? why not?
+      local_res_path = "#{dest_dir}/#{res_relative_path}"
+      puts "[debug] local_res_path: >#{local_res_path}<"
+      local_res_dir   = File.dirname( local_res_path )
+      FileUtils.mkdir_p( local_res_dir )
+
+      @worker.copy( res_url, local_res_path )
+    end
+  end
+
+end # class Downloader
+
+end # module CsvPack

data/lib/csvpack/pack.rb

@@ -5,6 +5,47 @@
 
 module CsvPack
 
+
+
+class Meta   ## Pack(age) Meta / Manifest / Descriptor
+  extend Forwardable
+
+  def self.load_file( path )
+    text = File.open( path, 'r:utf-8' ).read
+    load( text )
+  end
+  ## todo: add alias method read
+
+  def self.load( text )
+    hash = JSON.parse( text )
+    new( hash )
+  end
+  ## todo: add alias method parse
+
+
+  def initialize( h )
+    @h = h
+  end
+
+  def name()      @h['name']; end
+  def title()     @h['title']; end
+  def license()   @h['license']; end
+
+  ## todo/fix: wrap resource in a class - why? why not?
+  def resources() @h['resources']; end
+
+  ##############
+  def_delegators :@h, :[]    ## todo/fix: add some more hash delgates - why? why not?
+
+  def pretty_print( printer )
+    printer.text "Meta<#{object_id} @h.name=#{name}, ...>"
+  end
+end  # class Meta
+
+
+
+
+
 class Pack
   ## load (tabular) datapackage into memory
   def initialize( path )
@@ -13,16 +54,15 @@ class Pack
     ## - check: if path is a folder/directory
     ##    (auto-)add  /datapackage.json
 
-    text = File.open( path, 'r:utf-8' ).read
-    @h = JSON.parse( text )
+    @meta = Meta.load_file( path )
 
     pack_dir = File.dirname(path)
 
-    ## pp @h
+    pp @meta
 
     ## read in tables
     @tables = []
-    @h['resources'].each do |r|
+    @meta.resources.each do |r|
       ## build table data
       @tables << build_tab( r, pack_dir )
     end
@@ -30,9 +70,8 @@ class Pack
     ## pp @tables
   end
 
-  def name()    @h['name']; end
-  def title()   @h['title']; end
-  def license() @h['license']; end
+  def meta() @meta; end  ## delegate known meta props (e.g. name, title, etc. - why? why not?)
+
 
   def tables()  @tables; end
   ## convenience method - return first table
@@ -189,6 +228,7 @@ class Tab
     'datetime' => :datetime,
     'date'     => :date,
     'time'     => :time,
+    'year'     => :string,     ## note: map year for now to string - anything better? why? why not?
   }
 
   def dump_schema

data/lib/csvpack/version.rb

@@ -1,22 +1,22 @@
-# encoding: utf-8
-
-module CsvPack
-
-  MAJOR = 0 ## todo: namespace inside version or something - why? why not??
-  MINOR = 1
-  PATCH = 0
-  VERSION = [MAJOR,MINOR,PATCH].join('.')
-
-  def self.version
-    VERSION
-  end
-
-  def self.banner
-    "csvpack/#{VERSION} on Ruby #{RUBY_VERSION} (#{RUBY_RELEASE_DATE}) [#{RUBY_PLATFORM}]"
-  end
-
-  def self.root
-    File.expand_path( File.dirname(File.dirname(File.dirname(__FILE__))) )
-  end
-
-end  # module CsvPack
+# encoding: utf-8
+
+module CsvPack
+
+  MAJOR = 0 ## todo: namespace inside version or something - why? why not??
+  MINOR = 2
+  PATCH = 0
+  VERSION = [MAJOR,MINOR,PATCH].join('.')
+
+  def self.version
+    VERSION
+  end
+
+  def self.banner
+    "csvpack/#{VERSION} on Ruby #{RUBY_VERSION} (#{RUBY_RELEASE_DATE}) [#{RUBY_PLATFORM}]"
+  end
+
+  def self.root
+    File.expand_path( File.dirname(File.dirname(File.dirname(__FILE__))) )
+  end
+
+end  # module CsvPack

data/test/helper.rb

@@ -1,7 +1,7 @@
-
-## minitest setup
-require 'minitest/autorun'
-
-
-## our own code
-require 'csvpack'
+
+## minitest setup
+require 'minitest/autorun'
+
+
+## our own code
+require 'csvpack'