gphys 1.2.2.1 → 1.4.3

data/lib/numru/gphys/gphys_netcdf_io.rb

@@ -525,7 +525,7 @@ module NumRu
 	  # Single file. Returns a VArrayNetCDF. dim is ignored.
 	  file = files
 	  var = file.var( varname )
-	  raise "variable '#{varname}' not found in #{file}" if !var
+	  raise "variable '#{varname}' not found in #{file.inspect}" if !var
 	  VArrayNetCDF.new( var )
 	elsif files.is_a?(NArray)
 	  # Suppose that files is a NArray of NetCDF. Returns a VArrayCompsite.
@@ -566,9 +566,30 @@ module NumRu
 	[files, ncvar0]
       end
 
+      #--
+      #  re-arrange the array of files and check some inconsistencies
+      #
+      #  e.g.)
+      #     files = NArray[ file0.nc, file1.nc, file2.nc, file3.nc ]
+      #     The files have a variable, u(lat,lev,time),
+      #     which is tiled to four parts as the followings:
+      #                |    lat  |     lev    | time |
+      #       file0.nc | -90...0 | 1000...500 | 0..1 |
+      #       file1.nc |   0..90 | 1000...500 | 0..1 |
+      #       file2.nc | -90...0 |  500..100  | 0..1 |
+      #       file3.nc |   0..90 |  500..100  | 0..1 |
+      #
+      #     Then the output is
+      #        NArray[ [ [ file0.nc, file1.nc ],
+      #                  [ file2.nc, file3.nc ] ] ].
+      #
+      #  The order of the elements of the input array is arbitrary.
+      #
       def __files_dim_matching( files, varname )
 	# files: NArray of NetCDF
 
+	return files if files.length == 1
+
 	# < read the first file and check its rank >
 
 	file0 = files[0]
@@ -581,44 +602,61 @@ module NumRu
 
 	#< find correspoding dimensions >
 
-	j2i = Array.new
-
-	for ifl in 0...files.rank
-	  for jdt in 0...ncvar0.rank
-	    axpos_last = nil
-	    for kfl in 0...files.shape[ifl]
-	      nvax = files[*([0]*ifl+[kfl]+[0]*(files.rank-ifl-1))].var(axposnames[jdt])
-	      raise "No coordinate variable: #{axposname}' not found" if !nvax
-	      axpos = VArrayNetCDF.new( nvax )
-	      if kfl > 0
-		if axpos.length != axpos_last.length
-		  # not equal ==> this is the dimension looking for
-		  j2i[jdt] = ifl
-		  break
-		else
-		  axv0 = axpos.val[0]
-		  axv_last0 = axpos_last.val[0]
-		  if axv0 != axv_last0
-		    # not equal ==> this is the dimension looking for
-		    j2i[jdt] = ifl
-		    break
-		  end
-		end
-	      end
-	      axpos_last = axpos
-	    end
-	    break if j2i.include?(ifl)
-	  end
-	  if files.shape[ifl]>1 && !j2i.include?(ifl)
-	    raise "No dimension correpodence found for #{ifl}th dim" 
-	  end
-	end
-
-	for d in files.rank...ncvar0.rank
-	  j2i[ (j2i.index(nil) || files.rank) ] = d
-	  files.newdim!(d)
-	end
-	files = files.transpose(*j2i)
+        rank = axposnames.length
+        cvals = Array.new(rank){ Array.new }
+        sign = Array.new(rank)
+        units = Array.new(rank)
+        files.each do |file|
+          axposnames.each_with_index do |axposname, i|
+            nvax = file.var(axposname)
+            raise "No coordinate variable: #{axposname}' not found" if !nvax
+            axpos = VArrayNetCDF.new( nvax )
+            axv0 = axpos.val[0]
+            if units[i]
+              un = axpos.units
+              axv0 = un.convert2(axv0,units[i]) unless un == units[i]
+            else # the first file
+              units[i] = axpos.units
+              sign[i] = (axpos.val[-1] >= axv0 ? 1 : -1)
+            end
+            cvals[i].push axv0
+          end
+        end
+        # For the above example,
+        # cvals = [ [  -90,    0, -90,   0 ],
+        #           [ 1000, 1000, 500, 500 ],
+        #           [    0,    0,   0,   0 ] ]
+        # sign =  [ 1, -1, 1 ]
+
+        axs = Array.new(rank)
+        cvals.each_with_index do |cval, i|
+          axs[i] = cval.uniq
+          axs[i].sort!{|a,b| (a <=> b)*sign[i] }
+        end
+        # For the above example,
+        # axs   = [ [ -90, 0 ], [ 1000, 500 ], [ 0 ] ]
+        files_new = NArray.object( *axs.map{|ax| ax.length} )
+        # For the above example,
+        # shape of the files_new is [2, 2, 1]
+        files.length.times do |n|
+          file = files[n]
+          idx = Array.new(rank)
+          rank.times do |i|
+            axv0 = cvals[i][n]
+            idx[i] = axs[i].index(axv0)
+          end
+          # For the above example,
+          # idx =
+          #       [ 0, 0, 0 ] for n==0
+          #       [ 1, 0, 0 ] for n==1
+          #       [ 0, 1, 0 ] for n==2
+          #       [ 1, 1, 0 ] for n==3
+          files_new[*idx] = file
+        end
+        if files_new.eq(nil).count_true > 0
+          raise "No dimension correspondence found"
+        end
+        files = files_new
 
 	files
       end

data/lib/numru/gphys/gphys_nusdas_io.rb

@@ -34,6 +34,9 @@ module NumRu
         unless NuSDaS === nusdas
           raise ArgumentError, "1st arg must be a NuSDaS of path name"
         end
+        unless (meta = nusdas.instance_variable_get(:@meta)) && meta[:nbasetime] && meta[:nbasetime] > 0
+          raise "NuSDaS directory is empty or nusdas_def is inconsistent with data: #{nusdas.path}"
+        end
 
         var = nusdas.var(varname)
         var.nil? && raise("#{varname} is not found")

data/lib/numru/gphys/grib.rb

@@ -407,6 +407,12 @@ class GPhys
               params = PARAMS_JMA
             end
           end
+        when 200
+          if cid == 34 # JMA
+            params = PARAMS_JRA55
+          else
+            warn "parameter table version #{version}, center id #{cid} has not been defined yet"
+          end
         else
           if cid==98 # ECMWF
             if version==128
@@ -810,7 +816,7 @@ class GPhys
           vlat = y*1000
           grid[4,3] = int2str(vlat[0],3)
           grid[7,3] = int2str(vlon[0],3)
-          flag = 1<<7
+          flag = 1 << 7
           grid[10,1] = uint2str(flag,1)
           grid[11,3] = int2str(vlat[-1],3)
           grid[14,3] = int2str(vlon[-1],3)
@@ -823,7 +829,7 @@ class GPhys
             raise "not defined yet"
           end
           if rev
-            flag=1<<5
+            flag=1 << 5
           else
             flag=0
           end
@@ -963,7 +969,7 @@ class GPhys
         end
       end
       def get
-        @gds && uint2str(length,3)<<@gds
+        @gds && uint2str(length,3) << @gds
       end
     end # end definition of class GribGDS
     class GribBMS
@@ -1008,12 +1014,12 @@ class GPhys
       def set_map(mask)
         map,n = data2str(mask,1,1)
         @bms[0..0] = uint2str(n,1)
-        @bms = @bms[0,3]<<map
+        @bms = @bms[0,3] << map
         @sgm.is.update_total_length
         return true
       end
       def get
-        @bms && uint2str(length,3)<<@bms
+        @bms && uint2str(length,3) << @bms
       end
     end # end definition of class GribBMS
     class GribBDS
@@ -1194,7 +1200,7 @@ class GPhys
         @efactor = 2.0**str2int2( @sgm.file.read(2) )
         return @efactor
       end
-      def value(shape,*arg)
+      def value(shape,index=nil)
         nbits_unuse = oct4&15
         @sgm.file.seek(@pos+7, ::IO::SEEK_SET)
         nbits_pack = @sgm.file.read(1).to_uint1
@@ -1215,8 +1221,7 @@ class GPhys
           nlen = ((length-3-8)*8-nbits_unuse)/nbits_pack
           nlon,nlat = shape
           (nlon*nlat)==nlen+nmiss || raise("length is not collect")
-          if arg.length!=0
-            index, shape2 = arg
+          if index
             index = index.collect{|el| Fixnum===el ? el..el : el}
             il = index[0]
             i0 = il.first
@@ -1230,10 +1235,10 @@ class GPhys
             i0 = i0 < 0 ? nlat+i0 : i0
             i1 = i1 < 0 ? nlat+i1 : i1
             index[1] = il.exclude_end? ? i0...i1 : i0..i1
-            j = nil
-            i = nil
+            #j = nil
+            #i = nil
             if map
-              var = NArrayMiss.sfloat(*shape2)
+              var = NArrayMiss.sfloat(index[0].count, index[1].count)
               map.reshape!(nlon,nlat)
               mask = map[*index]
               nd = mask.count_true
@@ -1323,13 +1328,13 @@ class GPhys
         a = 127 if a>127
         b = ref*2**24*16**(64-a)
         a = a*s
-        @bds[3,4] = int2str(a,1)<<uint2str(b,3)
+        @bds[3,4] = int2str(a,1) << uint2str(b,3)
         @bds[7,1] = uint2str(nbits,1)
-        @bds = @bds[0,8]<<str
+        @bds = @bds[0,8] << str
         @sgm.is.update_total_length
       end
       def get
-        uint2str(length,3)<<@bds
+        uint2str(length,3) << @bds
       end
     end # end definition of class GribBDS
     class GribES
@@ -1366,7 +1371,7 @@ class GPhys
 
     public
 =begin
-=NumRu::Grib  -- a class for Grib datasets
+=NumRu::GPhys::Grib  -- a class for Grib datasets
 
 ==Class Methods
 ---Grib.new(filename, mode="r")
@@ -1555,7 +1560,7 @@ class GPhys
   end # end of definition of class Grib
 
 =begin
-=NumRu::GribDim
+=NumRu::GPhys::GribDim
 
 ==Class Methods
 ---new( vat, name, length )
@@ -1611,10 +1616,25 @@ class GPhys
         return @ary
       end
     end
-    def [](*ind)
-      return val[*ind]
+    def [](ind)
+      v = val
+      if Hash === ind    # {range=>step} --> array of indices
+        ran, step = ind.to_a[0]
+        dlen = v.length
+        f = ran.first
+        f += dlen if f<0
+        e = ran.end
+        e += dlen if e<0
+        e -= 1 if ran.exclude_end?
+        ind = Array.new
+        (f..e).step(step){|n| ind.push(n)}
+      end
+      return v[ind]
     end
     def put(ary)
+      if ary.typecode <= NArray::INT         # 2014-09-22 horinouchi
+        ary = ary.to_type(NArray::DFLOAT)
+      end 
       @ary = ary
       @length = val.length
       return @ary
@@ -1635,9 +1655,13 @@ class GPhys
   end
 
 =begin
-=NumRu::GribVar
+=NumRu::GPhys::GribVar
 
 ==Class Methods
+---always_time_seq=(t_or_f)
+If true is given, a GribVar will always have a time dimension,
+even when the time is unique (i.e., length of the time coordinate is 1).
+
 ---new( file, name, obj, dims )
 
 ==Methods
@@ -1666,6 +1690,15 @@ class GPhys
     include GribUtils
     class << self
       include GribUtils
+
+      @@always_time_seq = false
+      def always_time_seq=(t_or_f)
+        @@always_time_seq = t_or_f
+      end
+      def always_time_seq
+        @@always_time_seq
+      end
+
       private
       def vars_same?(var1,var2)
         pds1 = var1.pds
@@ -1816,7 +1849,7 @@ class GPhys
 #            time.push( get_time(sgm.pds.date) )
 #          }
           time.uniq!
-          if time.length>1
+          if @@always_time_seq or time.length>1
             time = NArray.to_na(time)
             d = va.def_dim("time",-1)
             d.put(time)
@@ -1909,55 +1942,96 @@ class GPhys
         return @obj[*indices]
       else
         sha = shape
+        ndims = sha.length
+        after_slice = false
+        dimdel = nil
         if indices.length!=0
+          after_slicer = Array.new(2,true)
+          imap = (3..ndims).collect{Hash.new}  # index map org-->current for z,t
           if indices[0] == false
-            indices[0] = [true]*(sha.length-indices.length+1)
+            indices[0] = [true]*(ndims-indices.length+1)
           elsif indices[-1] == false
-            indices[-1] = [true]*(sha.length-indices.length+1)
+            indices[-1] = [true]*(ndims-indices.length+1)
           elsif indices.include?(false)
             raise "invalid indices"
-          elsif sha.length!=indices.length
+          elsif ndims!=indices.length
             raise "invalid indices"
           end
-          sha.length.times{|n|
+          dimdel = []
+          ndims.times{|n|
             ind = indices[n]
-            if ind==true
-              indices[n] = 0..sha[n]-1
+            dlen = sha[n]
+            case ind
+            when true
+              if n<=1
+                indices[n] = 0..dlen-1
+              else
+                (0..dlen-1).each{|i| imap[n-2][i] = i}
+              end
               next
-            elsif Fixnum===ind
+            when Fixnum
               sha[n] = 1
+              imap[n-2][ind] = 0
+              dimdel.push(n)
               next
-            elsif Range===ind
+            when Range
               f = ind.first
+              f += dlen if f<0
               e = ind.end
-              e = sha[n]-1 if e==-1
+              e += dlen if e<0
               e -= 1 if ind.exclude_end?
               sha[n] = e-f+1
-              indices[n] = f..e
+              if n<=1
+                indices[n] = f..e
+              else
+                (f..e).each{|i| imap[n-2][i] = i-f}
+              end
               next
+            when Hash
+              ran, step = ind.to_a[0]
+              f = ran.first
+              f += dlen if f<0
+              e = ran.end
+              e += dlen if e<0
+              e -= 1 if ran.exclude_end?
+              sha[n] = (e-f)/step + 1
+              if n<=1
+                indices[n] = f..e
+                after_slice = true
+                after_slicer[n] = (0...sha[n]).collect{|i| i*step}
+              else
+                ia = NArray.int(sha[n]).indgen! * step + f
+                indices[n] = NArray.to_na(ia)
+                j=0
+                (f..e).step(step){|i| imap[n-2][i] = j; j+=1} if n>=2
+              end
+              next
+            when Array, NArray
+              ind = ind.collect{|i|
+                  raise("Array indices must be integers") unless Integer===i
+                  i>=0 ? i : i+dlen
+              }
+              sha[n] = ind.length
+              if n<=1
+                f = ind.min
+                e = ind.max
+                indices[n] = f..e
+                after_slice = true
+                after_slicer[n] = ind.collect{|i| i-f}
+              else
+                (0...ind.length).each{|i| imap[n-2][ind[i]] = i }
+              end
             else
               raise "invalid indices"
             end
           }
           if rank>2
-            mask = NArray.byte(*shape[2..-1])
-            mask[*indices[2..-1]]= 1
-            first = Array.new(indices.length-2)
-	    first.length.times{|i|
-              ind = indices[2+i]
-              if Fixnum===ind
-	        first[i] = ind < 0 ? shape[2+i]+ind : ind
-              elsif Range===ind
-                first[i] = ind.first
-              else
-                raise "invalid indices"
-              end
-            }
+            subsetzt = true
           else
-            mask = true
+            subsetzt = false
           end
         else
-          mask = true
+          subsetzt = false
         end
         value = @miss ? NArrayMiss.sfloat(*sha) : NArray.sfloat(*sha)
         if rank==2
@@ -1985,24 +2059,29 @@ class GPhys
           if vt
             index[-1] = vt.eq(get_time(pds.date)).where[0]
           end
-          next if (NArray===mask && mask[*index]==0)
+          if subsetzt
+            to_read = true
+            index.each_with_index{|i,d| to_read &&= imap[d][i]}
+            next unless to_read
+          end
           bds = sgm.bds
           if indices.length==0 || (indices[0]==true&&indices[1]==true)
             val = bds.value(shape[0..1])
           else
-            val = bds.value(shape[0..1],indices[0..1],sha[0..1])
+            val = bds.value(shape[0..1],indices[0..1])
           end
           d = pds.dfact
           val = val*10.0**(-d)
-          if NArray === mask
-            index.length==first.length || raise("invalide indices")
-            index.length.times{|i|
-              index[i] = index[i]-first[i]
-            }
+          val = val[*after_slicer] if after_slice
+          if !subsetzt
+            idx = index
+          else
+            idx = []
+            index.each_with_index{|i,d| idx.push(imap[d][i])}
           end
-          value[true,true,*index] = val
+          value[true,true,*idx] = val
         }
-        sha.delete(1)
+        dimdel.reverse_each{|d| sha.delete_at(d)} if dimdel
         value.reshape!(*sha)
         return value
       end