gfdnavi 1.3.0.0

data/lib/opendapdir.rb

@@ -0,0 +1,171 @@
+require 'open-uri'
+require 'numru/gphys'
+require 'date'
+
+module NumRu
+  class OPeNDAPDir
+    def initialize(url)
+      @path = url.sub(/\/$/,'')
+      @child_dirs = Array.new
+      @child_gphys_files = Array.new   # files that can have gphysiable data
+      @child_plain_files = Array.new     # external files (non-opendap access)
+      open(url) {|f|
+	if f.content_type != 'text/html'
+         raise(ArgumentError,"#{url}: Not text/html. Cannot be an OPeNDAP Dir")
+	end
+	opendapdir = false  # to initialize
+	while(line = f.gets)
+	  if /Parent Directory/i =~ line
+	    opendapdir = true
+	    break
+	  end
+	end
+	if !opendapdir
+	  raise(ArgumentError,"#{url}: is not likely an OPeNDAP directory.")
+	end
+	while(line = f.gets)
+	  case line
+	  when /<A HREF="(.*)">(.*)<\/a>\s*(.*)$/
+            child = { :url => $1 }
+	    label = $2
+            time_size_descr = $3
+            if /^(\S+\s+\S+)\s+(\S+)/ =~ time_size_descr
+              size = $2
+              time = $1
+              t = DateTime.parse(time)
+              child[:mtime] = Time.local(t.year,t.mon,t.day,t.hour,t.min,t.sec)
+                                  #^ local works with mysql (how about others?)
+              if /^([\d\.]+)([KMG])?$/ =~ size
+                if $2.nil?
+                  child[:size] = $1.to_i
+                else
+                  size = $1.to_f
+                  size *= 1000 if $2=='K'
+                  size *= 1000000 if $2=='M'
+                  size *= 10000000 if $2=='G'
+                  child[:size] = size.to_i
+                end
+              elsif '-' == size
+                child[:size] = nil
+              else
+                raise "(BUG) new pattern to support : #{size}"
+              end
+            end
+	    /^(.*)\/([^\/]+)\/?$/ =~ child[:url]
+	    dir = $1
+	    child[:name] = $2
+	    if /\/$/ =~ child[:url]
+	      @child_dirs.push(child)
+	    elsif /\.html$/ !~ label and /\.html$/ =~ child[:name]
+	      child[:url].sub!(/\.html$/,'')
+	      child[:name].sub!(/\.html$/,'')
+	      if /\.nc$/ =~ label  # we don't need to limit to .nc files, but..
+		@child_gphys_files.push(child)
+	      end
+	    elsif dir != @path
+	      @child_plain_files.push(child)
+	    else
+	      raise "(BUG) Unclassified kind of a child : #{child[:url]}"
+	    end
+#	    p child
+#	    print "#{File.basename(child[:url])}  #{child[:mtime]}  ",child[:size],"\n"
+#	    print "@@\n"
+	  end
+	end
+      }
+    end
+
+    class << self
+      alias open new
+    end
+
+    def path
+      @path
+    end
+
+    def name
+      File.basename(path)
+    end
+
+    def entries
+      if @enttries.nil?
+        @enttries = @child_dirs.collect{|c| c[:name]} +
+          @child_gphys_files.collect{|c| c[:name]} +
+          @child_plain_files.collect{|c| c[:name]}
+      end
+      @enttries
+    end
+
+    def each_dir
+      @child_dirs.each{|child|
+	dir = OPeNDAPDir.new(child[:url])
+	yield(dir, child[:mtime])
+      }
+      nil
+    end
+
+    # Open dods url as NetCDF files and iterate.
+    # It is up to the users to close the files.
+    def each_gphys_file
+      @child_gphys_files.each{|child|
+        begin
+          file = NetCDF.open(child[:url])
+          yield(file.path, child[:mtime], child[:size], file.class)
+        rescue NetcdfSyserr
+          warn "#{__FILE__}:#{__LINE__}: **Error neglected** Unable to open #{child[:url]}"
+        end
+      }
+      nil
+    end
+
+    # name_pattern : regexp to limit by name
+    #
+    def each_file(name_pattern = nil)
+      @child_plain_files.each{|child|
+        if name_pattern.nil? or name_pattern =~ child[:name]
+          yield(child[:url], child[:mtime], child[:size])
+        end
+      }
+    end
+
+    #def plain_file_paths
+    #  @child_plain_files.collect{|child| child[:url]}
+    #end
+
+  end
+end
+
+if __FILE__ == $0
+  include NumRu
+  urls = %w(http://coastwatch.chesapeakebay.noaa.gov/dap/edac/qscat_wind/data/
+            http://coastwatch.chesapeakebay.noaa.gov/dap/edac/qscat_wind/
+            http://stellwagen.er.usgs.gov/cgi-bin/nph-dods/DATAFILES/ECOHAB_II/
+            http://dods.ipsl.jussieu.fr/cgi-bin/nph-dods/prism/gridsCF/
+            http://test.opendap.org/opendap-3.7/nph-dods/data/ )
+#  urls = %w( http://dust.ess.uci.edu/cgi-bin/dods/nph-dods/dodsdata )
+#  urls = %w( http://test.opendap.org/opendap-3.7/nph-dods/data/ )
+
+  urls.each do |url|
+    dir = OPeNDAPDir.new(url)
+    print dir.path,"\n"
+    dir.each_dir{|d,mtime| print "  (sub dir) #{d.path} #{mtime}\n"}
+
+    count = 0
+    dir.each_gphys_file do |file, mtime, size|
+      print "  #{file.path} #{mtime} size:#{size}\n    "
+      p GPhys::IO.var_names_except_coordinates(file)
+      file.close
+      count += 1
+      break if count >= 3
+    end
+
+    pat = Regexp.union(/readme$/i, /\.txt$/)
+    dir.each_file(pat) do |path, mtime, size|
+      open(path){ |file| 
+        print " ***** ",path, " ", mtime, " ", size, "\n"
+        file.gets && print($_) && file.gets && print($_)
+      }
+    end
+  end
+
+end

data/lib/tar.rb

@@ -0,0 +1,127 @@
+require "zlib"
+
+module Tar
+
+  Header = [
+    ["name"    , 100],
+    ["mode"    ,   8],
+    ["uid"     ,   8],
+    ["gid"     ,   8],
+    ["size"    ,  12],
+    ["mtime"   ,  12],
+    ["chksum"  ,   8],
+    ["typeflag",   1],
+    ["linkname", 100],
+    ["magic"   ,   6],
+    ["version" ,   2],
+    ["uname"   ,  32],
+    ["gname"   ,  32],
+    ["devmajor",   8],
+    ["devminor",   8],
+    ["prefix"  , 155]
+   ]
+
+  TMAGIC = "ustar "
+  TVERSION = " "
+
+  REGTYPE = 0
+  DIRTYPE = "5"
+
+  def cf(io, files)
+    lclose = false
+    if String === io
+      io = File.open(io,"wb")
+      lclose = true
+    elsif ! IO === io
+      raise "the first argument must be IO or String"
+    end
+    begin
+      files.each{|name|
+        type = name[-1..-1] == "/" ? "d" : "f"
+        if type == "f"
+          size = File.stat(name).size
+        else
+          size = 0
+        end
+        io.write create_header(name, type, size)
+        if type == "f"
+          File.open(name,"rb"){|file|
+            while (buf = file.read(512))
+              io.write buf
+              io.write "\000"*(512 - buf.length) unless buf.length == 512
+            end
+          }
+        end
+      }
+      io.write "\000"*1024
+    ensure
+      io.close if lclose
+    end
+    return nil
+  end
+  module_function :cf
+
+  def zcf(io, files)
+    if IO === io
+      gz = Zlib::GzipWriter.new(io)
+      lclose = false
+    elsif String === io
+      gz = Zlib::GzipWriter.open(io)
+      lclose = true
+    else
+      raise "the first argument must be IO or String"
+    end
+    begin
+      cf(gz, files)
+    ensure
+      lclose ? gz.close : gz.finish
+    end
+    return nil
+  end
+  module_function :zcf
+
+  def create_header(name, type, size)
+    hash = Hash.new
+    hash["name"] = name
+    case type
+    when "f"
+      hash["mode"] = "0100644"
+      hash["typeflag"] = "%i"%REGTYPE
+    when "d"
+      hash["mode"] = "0000755"
+      hash["typeflag"] = "%i"%DIRTYPE
+    end
+    hash["uid"] = "0001000"
+    hash["gid"] = "0001000"
+    hash["size"] = "%0#11o"%size
+    hash["mtime"] = ("%#o"%Time.now.to_i)[1..-1]
+    hash["linkname"] = ""
+    hash["magic"] = TMAGIC
+    hash["version"] = TVERSION
+    hash["uname"] = "gfdnavi"
+    hash["gname"] = "gfdnavi"
+    hash["devmajor"] = ""
+    hash["devminor"] = ""
+    hash["prefix"] = ""
+    hash["chksum"] = " "*8
+    chksum = 0
+    hash.each{|n,v|
+      v.each_byte{|byte| chksum +=  byte }
+    }
+    chksum = "%#o"%chksum
+    len = chksum.size
+    hash["chksum"][0...len] = chksum
+    hash["chksum"][len..len] = "\000"
+
+    header = ""
+    Header.each{|ary|
+      str = hash[ary[0]]
+      str += "\000"*(ary[1]-str.length) if ary[1] > str.length
+      header += str
+    }
+    header += "\000"*(512-header.length%512)
+    return header
+  end
+  module_function :create_header
+
+end

data/lib/tasks/clean.rake

@@ -0,0 +1,65 @@
+desc "Clean all"
+task :clean do
+  Rake::Task["clean:diagram_cache"].invoke
+  Rake::Task["clean:session_files"].invoke
+  Rake::Task["clean:work_dirs"].invoke
+end
+
+namespace :clean do
+
+  desc "Clean diagram cache"
+  task :diagram_cache => :environment do
+    path = GFDNAVI_DIAGRAM_CACHE_PATH + "/"
+    Dir.foreach(path){|fname|
+      fname = path + fname
+      File.delete(fname) unless File.directory?(fname)
+    }
+    connection = ActiveRecord::Base.connection
+    tables = connection.tables
+    if tables.include?("diagram_cache_sessions")
+      DiagramCacheSession.destroy_all unless DiagramCacheSession.count == 0
+    end
+    if tables.include?("diagram_cache_data")
+      DiagramCacheDatum.destroy_all unless DiagramCacheDatum.count == 0
+    end
+    if tables.include?("diagram_caches")
+      DiagramCache.destroy_all unless DiagramCache.count == 0
+    end
+  end
+
+  desc "Clean session files"
+  task :session_files do
+    session_dir = "#{RAILS_ROOT}/tmp/sessions"
+    Dir.foreach(session_dir){|fname|
+      if fname =~ /^ruby_sess\./
+        File.delete("#{session_dir}/#{fname}")
+      end
+    }
+  end
+
+  except_dirs = %w(. .. CVS)
+
+  desc "Clean work directories"
+  task :work_dirs => :environment do
+    path = GFDNAVI_WORK_PATH + "/"
+    Dir.foreach(path){|dname|
+      next if except_dirs.include?(dname)
+      dname = path + dname + "/"
+      next unless File.directory?(dname)
+      Dir.foreach(dname){|fname|
+        begin
+          File.delete(dname + fname) unless except_dirs.include?(fname)
+        rescue
+          # Suppose that fname is a directory
+          Dir.foreach(dname + fname){|subfname|
+             File.delete(dname + fname +'/' + subfname) \
+                                 unless except_dirs.include?(subfname)
+          }
+          Dir.delete(dname + fname)
+        end
+      }
+      Dir.delete(dname)
+    }
+  end
+
+end

data/lib/tasks/setup.rake

@@ -0,0 +1,316 @@
+desc "Setup all"
+
+task :setup_noquestions do
+  $noquestions = true
+  Rake::Task["setup:tables"].invoke
+  # UPDATE NODE INFO
+  NodeEntityAbstract.define_name_methods
+  Rake::Task["setup:tree"].invoke
+  Rake::Task["setup:functions"].invoke
+  Rake::Task["setup:draw_methods"].invoke
+end
+
+task :setup => :environment do
+  if exist_tables.include?("nodes") && Node.count > 0
+    print "This will clear all the database tables and caches.\n"
+    print "If you want just to rebuild database tables, use \"rake update\",\n"
+    print "                 or update tree,             use \"rake update:tree\",\n"
+    print "instead.\n"
+    print "Are you sure to continue? (no/yes, default:no) "
+    res = $stdin.gets.chomp!
+    unless res == "yes"
+      abort "execution was canceled"
+    end
+  end
+  Rake::Task["clean"].invoke
+  Rake::Task["setup:tables"].invoke
+  # UPDATE NODE INFO
+  Rake::Task["setup:tree"].invoke
+  Rake::Task["setup:functions"].invoke
+  Rake::Task["setup:draw_methods"].invoke
+end
+
+namespace :setup do
+  desc "Setup database tables"
+  task :tables => :environment do
+    command = 'ruby "' + File.join(File.expand_path(RAILS_ROOT), 'db', 'dump_db.rb') + '"'
+    unless system( command )
+      raise "failed to execute '#{command}'"
+    end
+    fname_user = File.join(RAILS_ROOT,"db","user.yml")
+    if File.exist?(fname_user)
+      unless $noquestions
+        print "Do you want to keep user information? (yes/no, default:yes): "
+        res = $stdin.gets.chomp!
+      else
+        res = "yes"
+      end
+      if res == "no"
+        suffix = Time.now.strftime("%Y%m%d%H%M%S")
+        File.rename(fname_user, fname_user+"."+suffix)
+        fname_group = File.join(RAILS_ROOT,"db","group.yml")
+        File.rename(fname_group, fname_group+"."+suffix) if File.exist?(fname_group)
+      end
+    end
+
+    connection = ActiveRecord::Base.connection
+    connection.drop_table("schema_info") if connection.tables.include?("schema_info")
+    connection.drop_table("schema_migrations") if connection.tables.include?("schema_migrations")
+    Rake::Task["db:migrate"].invoke
+
+    if File.exist?(fname_user)
+      Rake::Task["setup:user"].invoke
+    else
+      Rake::Task["setup:root_password"].invoke
+    end
+
+  end
+
+  desc "Set root password"
+  task :root_password => :environment do
+    unless /mswin(?!ce)|mingw|bccwin/ =~ RUBY_PLATFORM
+      system("stty -echo > /dev/null 2>&1") #for unix-like systems
+    end
+    while true
+      while true
+        print "Input password for root (number of characters must be 5 to 40): "
+        pw2 = $stdin.gets.chomp!
+        if pw2.length < 5
+          print "The password which you input is too short.\n"
+        elsif pw2.length > 40
+          print "The password which you input is too long.\n"
+        else
+          break
+        end
+      end
+      print "\nRepeat password: "
+      tmp = $stdin.gets.chomp!
+      break if tmp == pw2
+      print "\nNot correct!\n"
+    end
+    unless /mswin(?!ce)|mingw|bccwin/ =~ RUBY_PLATFORM
+      system("stty echo > /dev/null 2>&1") #for unix-like systems
+    end
+    print("\n")
+    if root = User.find_by_login("root")
+      if root.change_password(pw2,tmp)
+        print "changing was succeded\n"
+      else
+        print "changing was not succeded\n"
+      end
+    else
+      root = User.new( :login => "root",
+                       :full_name => "default super user",
+                       :email_address => GFDNAVI_ADMIN_EMAIL,
+                       :affiliation => "GFDNAVI",
+                       :password => pw2,
+                       :password_confirmation => pw2 )
+      root.super_user = true
+      root.internal_user = true
+      root.save!
+    end
+  end
+
+  desc "Create or restore User information"
+  task :user => :environment do
+    %w(user group).each{|table_name|
+      if exist_tables.include?(table_name)
+        model = ActiveRecord.class_eval(table_name.classify)
+        model.delete_all
+      end
+    }
+    fname = File.join(RAILS_ROOT,"db","user.yml")
+    if File.exist?(fname)
+      yml = YAML.load(File.read(fname))
+      raise("user data is invalid") unless Hash===yml
+      yml.each{|u,hash|
+        pw = hash.delete("password")
+	su = hash.delete("super_user")
+        user = User.new(hash)
+        pw_tmp = "temporary passowrd"
+        user.password = pw_tmp
+        user.password_confirmation = pw_tmp
+	user.super_user = su
+        User.transaction do
+          user.password = pw
+          user.instance_eval("create_without_callbacks") || raise(RecodeNotSaved)
+        end
+      }
+    end
+    fname = File.join(RAILS_ROOT,"db","group.yml")
+    if File.exist?(fname)
+      yml = YAML.load(File.read(fname))
+      raise("group data is invalid") unless Hash===yml
+      yml.each{|g,hash|
+        group = Group.new(hash)
+        group.save!
+      }
+    end
+  end
+
+  desc "Setup tree of directories and variables"
+  task :tree => :environment do
+    Rake::Task["clean:diagram_cache"].invoke
+    %w( draw_parameters
+      keyword_attributes spatial_and_time_attributes
+      node_relations node_lineages
+      knowledge_figures knowledges
+      actual_files images variables directories
+      nodes).each{|table_name|
+      if exist_tables.include?(table_name)
+        model = ActiveRecord.class_eval(table_name.classify)
+	model.delete_all
+      end
+    }
+    command = 'ruby "' + File.join(File.expand_path(RAILS_ROOT), 'db', 'register_directory_tree.rb') + '"'
+    unless system( command )
+      raise "failed to execute '#{command}'"
+    end
+  end
+
+  desc "Setup default functions database"
+  task :functions => :environment do
+    print "registeration functions\n"
+    create_value_type_table
+    unless Function.table_exists?
+      raise "functions table does not exist"
+    end
+    unless FunctionOutput.table_exists?
+      raise "function_outputs table does not exist"
+    end
+    unless FunctionArgument.table_exists?
+      raise "function_arguments table does not exist"
+    end
+    FunctionArgument.delete_all
+    FunctionOutput.delete_all if FunctionOutput.table_exists?
+    Function.delete_all if Function.table_exists?
+    Node.delete_all(['node_type = ?',Node::FUNCTION])
+    dir = File.join(File.expand_path(RAILS_ROOT), 'db', 'functions')
+    root = User.find_by_login("root")
+    regist_functions(dir, root, true)
+    Dir.foreach(GFDNAVI_USER_PATH){|dname|
+      user = User.find_by_login(dname)
+      dir = File.join(GFDNAVI_USER_PATH, dname, "functions")
+      if user && File.exist?(dir)
+        regist_functions(dir, user)
+      end
+    }
+  end
+
+  desc "Setup default draw methods database"
+  task :draw_methods => :environment do
+    print "registeration draw methods\n"
+    create_value_type_table
+    unless DrawMethod.table_exists?
+      raise "draw_methods table does not exist"
+    end
+    unless DrawMethodAttribute.table_exists?
+      raise "draw_method_options table does not exist"
+    end
+    DrawMethodAttribute.delete_all
+    DrawMethod.delete_all 
+    Node.delete_all(['node_type = ?',Node::DRAW_METHOD])
+    dir = File.join(File.expand_path(RAILS_ROOT), 'db', 'draw_methods')
+    root = User.find_by_login("root")
+    regist_draw_methods(dir, root, true)
+    Dir.foreach(GFDNAVI_USER_PATH){|dname|
+      user = User.find_by_login(dname)
+      dir = File.join(GFDNAVI_USER_PATH, dname, "draw_methods")
+      if user && File.exist?(dir)
+        regist_draw_methods(dir, user)
+      end
+    }
+  end    
+
+end
+
+
+
+def create_value_type_table
+  if ValueType.table_exists?
+    %w(int float string array_int array_float array_string boolean).each{|typ|
+      ValueType.new(:name => typ).save! unless ValueType.find_by_name(typ)
+    }
+  end
+end
+
+
+def regist_functions(dir, user, default=false)
+  path = "/usr/#{user.login}/functions"
+  Node.make_user_directory(path, user, default ? 4 : 0)
+  Dir.foreach(dir){|yml|
+    next unless /\.yml$/=~yml
+    fname = File.join(dir,yml)
+    fixture = YAML.load( File.read(fname) )
+    arguments = fixture.delete(:arguments)
+    outputs = fixture.delete(:outputs)
+    func = Function.new
+    fixture.each{|k,v|
+      func.send(k.to_s+"=",v)
+    }
+    func.path = File.join(path, func.name)
+    func.owner = user
+    func.default = default
+    func.other_mode = default ? 4 : 0
+    Function.transaction{
+      FunctionOutput.transaction{
+        FunctionArgument.transaction{
+          func.save!
+          func.function_outputs.create(outputs)
+          if arguments
+            arguments.each{|arg|
+              typ = arg.delete(:value_type)
+              fa = FunctionArgument.new(arg)
+              fa.value_type = ValueType.find_by_name(typ)
+              fa.function = func
+              fa.save!
+            }
+          end
+        }
+      }
+    }
+  }
+end
+
+
+
+def exist_tables
+  ActiveRecord::Base.connection.tables
+end
+
+
+def regist_draw_methods(dir, user, default=false)
+  path = "/usr/#{user.login}/draw_methods"
+  Node.make_user_directory(path, user, default ? 4 : 0)
+  Dir.foreach(dir){|yml|
+    next unless /\.yml$/=~yml
+    fname = File.join(dir,yml)
+    fixture = YAML.load( File.read(fname) )
+    attrs = fixture.delete(:attributes)
+    dm = DrawMethod.new
+    fixture.each{|k,v|
+      dm.send(k.to_s+"=",v)
+    }
+    dm.path = File.join(path, dm.name)
+    dm.owner = user
+    dm.default = default
+    dm.other_mode = default ? 4 : 0
+    DrawMethod.transaction{
+      DrawMethodAttribute.transaction{
+        dm.save!
+        if attrs
+          attrs.each{|attr|
+            typ = attr.delete(:type)
+            unless typ
+              raise "type must be set for attributes"
+            end
+            dma = DrawMethodAttribute.new(attr)
+            dma.value_type = ValueType.find_by_name(typ)
+            dma.draw_method = dm
+            dma.save!
+          }
+        end
+      }
+    }
+  }
+end