dir-to-xml 1.0.6 → 1.1.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: a24fbbc39d5a5d420a7b67713292409a71aae25b72484c9af7f261af3b1f952f
-  data.tar.gz: 8e33e3dd2579ee7ef0c3c56144b2cc2beecb68784e20c60b73ea1f8bb1adc9fb
+  metadata.gz: 372af34f7da498e42ef32c51ee16e6c4f4270f70363d937f577390e1c690ba04
+  data.tar.gz: 49f55f26145a5b0d43120f2a06601641ef1e1dc39acf1be24e9993d78421b333
 SHA512:
-  metadata.gz: c45dfed7498b83d0918456d4deb47570cec2f6ab63e27baf0cc26f815927a161a783d3278f6ad5bb421f7d199f9332ea42fd3e9afa10c8ba1c4bd17838785ca6
-  data.tar.gz: 0dc2bdf0cc9483abf4f0ec8fc4f33c48a2d64e9e30cc1b281383c1bb3f9586b7aa4bd5db3459fed4a205e1b9c05954bdaa6629c416f78e357fbe543e4261a154
+  metadata.gz: ff46b964f643d8d32d0f681596d1e8c4d193fe967775cfaf5502d52aa28ef6f906c5811d607ac9d9147ba1dae6374ba8b4704306ff1e679dead279e609d0f5a1
+  data.tar.gz: 65a8b57721dd15a038e9190e736355476d052d63f535c9386024bd5c01099870d8d01e0f80bc92ca7314e47fcf2d72f061af6c19a953463a3e174185ee9b0a00

data/lib/dir-to-xml.rb

@@ -2,272 +2,261 @@
 
 # file: dir-to-xml.rb
 
+require 'c32'
 require 'dxlite'
 
 
+# How dir-to-xml should work
+#
+# if the dir.xml file doesn't exist then
+#
+#   generate the dir.xml file
+#
+# else
+#
+#   # the dir.xml file does exist
+#
+#   # check for one of the following:
+#   #  1 or more new files
+#   #  1 or more removed files
+#   #  1 or more modified files
+#
+# note: Ideally The index needs to be stored and retrieved the fastest way possible.
+#       This is why it's saved as a .json file rather .xml
+#
+# tested:
+# * finding the latest file in the current directory
+# * finding the latest file in a sub-directory (using recursive: true)
+
+
 class DirToXML
+  using ColouredText
 
-  attr_reader :dx, :activity
-  
-  def initialize(x= '.', recursive: false, index: 'dir.xml', debug: false)
-    
-    super()
-    
-    @debug = debug
-    
-    @dx = nil
-    @activity = {new: [], modified: []}
-    
-    if x.is_a? DxLite then
-      
-      @dx = x      
-      @a = @dx.to_a    
-      @object = @a      
-      
-      return self
-    end
-    
-    path = x
-    
-    @path, @index, @recursive = path, index, recursive
-    
-    raise "Directory not found." unless File.exists? path
-    filepath = File.join(path, index)
-    
-    
-    if File.exists? filepath then
-    
-      @dx = DxLite.new(File.join(@path, @index), debug: @debug)
-      
-    else
-      
-      @dx = DxLite.new('directory[title, file_path, last_modified, ' + \
-              'description]/file(name, type, ext, ctime, mtime, atime, ' + \
-              'description, owner, group, permissions)')
-
-      puts 'before title' if @debug
-      @dx.title = 'Index of ' + File.expand_path(@path)
-      @dx.file_path = File.expand_path(@path)
-      @dx.last_modified = ''
-    
-    end        
-    
-    puts 'before Dir.glob' if @debug
-    
-    a = Dir.glob(File.join(path, "*")).map{|x| File.basename(x) }.sort
-        
-    a.delete index
-
-    a2 = a.inject([]) do |r, filename|
-
-      x = File.join(path, filename)
-      
-      begin
-      r << {
-        name: filename,
-        type: File::ftype(x),
-        ext: File.extname(x),
-        ctime: File::ctime(x),
-        mtime: File::mtime(x),
-        atime: File::atime(x)
-      }
-      rescue
-        r
-      end
+  attr_reader :new_files, :deleted_files, :dx, :latest_files, :latest_file
 
-    end    
-    
-    # has the directory been modified since last time?
-    #
-    if @dx and @dx.respond_to? :last_modified and \
-        @dx.last_modified.length > 0 then
-      
-      puts 'nothing to do' if @debug
-      
-      file = a2.max_by {|x| x[:mtime]}
-      
-      if @debug then
-        puts 'file: ' + file.inspect 
-        puts 'd1: '  + Time.parse(@dx.last_modified).inspect
-        puts 'd2: ' + (file[:mtime]).inspect
-      end
-      
-      return if Time.parse(@dx.last_modified) >= file[:mtime]
-      
-    end    
-    
-    puts 'stage 2'
-    
-    if @dx and @dx.respond_to? :last_modified  then
-      
-      if @dx.last_modified.length > 0 then
-      
-        t = Time.parse(@dx.last_modified)
-        
-        # find the most recently modified cur_files
-        recent = a2.select {|x| Time.parse(x[:mtime]) > t }.map {|x| x[:name]} \
-            - %w(dir.xml dir.json)
-        
-        # is it a new file or recently modified?
-        new_files = recent - @dx.to_a.map {|x| x[:name]}
-        modified = recent - new_files
-        
-      else
-        
-        new_files = a2.select {|x| x[:type] == 'file'}.map {|x| x[:name]}
-        modified = []
-        
-      end
-      
-      @activity = {modified: modified, new: new_files}
-      
+  def initialize(obj= '.', index: 'dir.json', recursive: false,
+                 verbose: true, debug: false)
+
+    if verbose then
+      puts
+      puts 'DirToXML at your service!'.highlight
+      puts
+      puts
     end
-      
 
-    command = File.exists?(File.join(path, index)) ? :refresh : :dxify
+    @index, @recursive, @verbose, @debug = index, recursive, verbose, debug
 
-    self.method(command).call a2
-    puts '@dx: ' + @dx.inspect if @debug
-    puts '@dx.last_modified: ' + @dx.last_modified.inspect if @debug
-    
-    
-    @a = @dx.to_a    
-    
-    if recursive then
+    if File.basename(obj) == index then
 
-      self.filter_by(type: :directory).to_a.each do |x|
+      #read the index file
+      @path = File.dirname(obj)
+      puts 'intialize() @path: ' + @path.inspect if @debug
 
-        path2 = File.join(path, x[:name])
-        DirToXML.new(path2, recursive: true)
-      end
+      @dx = read(obj)
+
+    else
+      @path = obj
+      puts 'intialize() @path: ' + @path.inspect if @debug
+
+      new_scan()
     end
-    
-    @object = @a
 
   end
-  
-  def filter(&blk)
-    @dx.filter &blk
+
+  def activity()
+    {
+      new: @new_files,
+      deleted: @deleted_files,
+      modified: @latest_files
+    }
   end
-  
-  def filter_by(pattern=/.*/, type: nil, ext: nil)
-    
-    @object = @a.select do |x| 
-            
-      pattern_match = x[:name] =~ pattern
-      
-      type_match = type ? x[:type] == type.to_s : true
-      ext_match = ext ? x[:ext] == ext.to_s : true
-
-      pattern_match and type_match and ext_match
 
-    end
+  alias changes activity
 
-    self
+  def directories()
+    @dx.all.select {|x| x.type == 'directory'}.map(&:name)
   end
-    
+
+  def find_all_by_ext(s)
+    @dx.find_all_by_ext(s)
+  end
+
   def find_by_filename(s)
-    @dx.all.find {|item| item.name == s}
+    @dx.find_by_filename(s)
   end
-  
-  alias find_by_file find_by_filename
-  
-  def last_modified(ext=nil)
 
-    if ext and ext != '*' then
-      @object = @a.select{|x| x[:ext][/#{ext}/] or x[:type] == 'directory'}
+  def latest()
+
+    if @latest_file then
+      File.join(@latest_file[:path], @latest_file[:name])
     end
-    
-    a = sort_by :mtime
 
-    lm =  a[-1]
-    
-    if @recursive and lm[:type] == 'directory' then
-      return [lm, DirToXML.new(File.join(@path,  lm[:name])).last_modified]
+  end
+
+  def new_scan()
+
+    t = Time.now
+    records = scan_dir @path
+    puts 'new_scan() records: ' + records.inspect if @debug
+
+    a = records.map {|x| x[:name]}
+
+    if File.exists? File.join(@path, @index) then
+
+      @dx = read()
+
+      old_records = @dx.to_a
+      a2 = old_records.map {|x| x[:name]}
+
+      # delete any old files
+      #
+      @deleted_files = a2 - a
+
+      if @deleted_files.any? then
+
+        @deleted_files.each do |filename|
+          record = @dx.find_by_name filename
+          record.delete if record
+        end
+
+      end
+
+      # check for newly modified files
+      # compare the file date with index file last modified date
+      #
+      dtx_last_modified = Time.parse(@dx.last_modified)
+
+      select_records = records.select do |file|
+        file[:mtime] > dtx_last_modified
+      end
+
+      find_latest(select_records)
+
+      # Add any new files
+      #
+      @new_files = a - a2
+
+      if @new_files.any? then
+
+        @dx.last_modified = Time.now.to_s
+        @dx.import @new_files.map {|filename| getfile_info(filename) }
+
+      end
+
+      @dx.last_modified = Time.now.to_s if @deleted_files.any?
+
     else
-      lm
+
+      @dx = new_index(records)
+      find_latest(records)
+
     end
+
+    t2 = Time.now - t
+    puts ("directory scanned in %.2f seconds" % t2).info if @verbose
+
   end
-  
-  def save()
-    @dx.save File.join(@path, @index)
-  end
-  
-  def select_by_ext(ext, &blk)
-    
-    @object = ext != '*' ? @a.select{|x| x[:ext][/#{ext}$/]} : @a
-    return if @object.empty?
-    
-    dx = DxLite.new
-    dx.import @object
-    dtx = DirToXML.new(dx)
-    block_given? ? dtx.dx.all.map(&:name).each(&blk) : dtx
-  end
-  
-  def sort_by(sym)
-    
-    puts 'inside sort_by' if @debug
-    procs = [[:mtime, lambda{|obj| obj.sort_by{|x| x[:mtime]}}]]
-    proc1 = procs.assoc(sym).last
-    
-    puts '@object: ' + @object.inspect if @debug
-    @object = @a = @dx.to_a if @object.nil?
-    proc1.call(@object)
-    
-  end
-  
-  def sort_by_last_modified()
-    sort_by :mtime
-  end
-  
-  alias sort_by_lastmodified sort_by_last_modified
-  
-  def to_a
-    @object || @a
-  end
-  
-  def to_h()
-    self.to_a.inject({}){|r,x| r.merge(x[:name] => x)}
-  end
-  
-  def to_xml(options=nil)
-    @dx.to_xml options
-  end
-  
-  def to_dynarex
-    @dx.clone
+
+  def read(index=@index)
+
+    t = Time.now
+    puts 'read path: ' + File.join(@path, index).inspect if @Debug
+
+    dx = DxLite.new(File.join(@path, index), autosave: true)
+
+    t2 = Time.now - t
+    puts ("%s read in %.2f seconds" % [@index, t2]).info if @verbose
+
+    return dx
+
   end
-  
-  alias to_dx to_dynarex
-  
+
   private
-      
-  def dxify(a)
-    
-    @dx.last_modified = Time.now.to_s  if @dx.respond_to? :last_modified
-    @dx.import a
-    @dx.save File.join(@path, @index)
 
-  end
+  def find_latest(files)
+
+    @latest_files = files.sort_by {|file| file[:mtime]}
+    puts '@latest_files:  ' + @latest_files.inspect if @debug
+
+    @latest_file = @latest_files[-1]
+    @latest_file[:path] = @path
+    puts ':@latest_file: ' + @latest_file.inspect if @debug
+
+    dir_list = directories()
+
+    if dir_list.any? then
+
+      dir_latest = dir_list.map do |dir|
+
+        puts 'dir: ' + dir.inspect if @debug
+        dtx2 = DirToXML.new(File.join(@path, dir), index: @index,
+                            recursive: true, verbose: false, debug: @debug)
+        [dir, dtx2.latest_file]
+
+      end.reject {|_,latest|  latest.nil? }.sort_by {|_, x| x[:mtime]}.last
+
+      puts 'dir_latest: ' + dir_latest.inspect if @debug
+
+      @latest_file = if dir_latest and \
+                          ((dir_latest.last[:mtime] > latest_file[:mtime]) \
+                                     or latest_file.nil? \
+                                     or latest_file[:type] == 'directory') then
 
-  def refresh(cur_files)
+        dir_latest.last[:path] = File.join(@path, dir_latest.first)
+        dir_latest.last
 
-    puts 'inside refresh' if @debug    
+      elsif latest_file and latest_file[:type] == 'file'
 
-    prev_files = @dx.to_a
-    
-    #puts 'prev_files: ' + prev_files.inspect
-    #puts 'cur_files: ' + cur_files.inspect
-            
-    cur_files.each do |x|
+        latest_file[:path] = @path
+        latest_file
 
-      file = prev_files.find {|item| item[:name] == x[:name] }
-      #puts 'found : ' + file.inspect if @debug
-      x[:description] = file[:description] if file and file[:description]
+      end
+
+    else
+      return
+    end
+
+  end
+
+  def getfile_info(filename)
+
+    x = File.join(@path, filename)
+    puts 'x: ' + x.inspect if @debug
+
+    begin
+      {
+        name: filename,
+        type: File::ftype(x),
+        ext: File.extname(x),
+        mtime: File::mtime(x),
+        description: ''
+      }
     end
+  end
+
+  def new_index(records)
+
+    dx = DxLite.new('directory[title, file_path,  ' +
+                      'last_modified, description]/file(name, ' +
+                                    'type, ext, mtime, description)')
+
+    puts 'before title' if @debug
+    dx.title = 'Index of ' + @path
+    dx.file_path = @path
+    dx.last_modified = Time.now
+    dx.import records.reverse
+    dx.save File.join(@path, @index)
+
+    return dx
+
+  end
+
+  def scan_dir(path)
+
+    a = Dir.glob(File.join(path, "*")).map {|x| File.basename(x) }
+    a.delete @index
+    a.map {|filename| getfile_info(filename) }
 
-    dxify(cur_files)
-    
   end
-  
+
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: dir-to-xml
 version: !ruby/object:Gem::Version
-  version: 1.0.6
+  version: 1.1.1
 platform: ruby
 authors:
 - James Robertson
@@ -11,52 +11,72 @@ cert_chain:
 - |
   -----BEGIN CERTIFICATE-----
   MIIEXjCCAsagAwIBAgIBATANBgkqhkiG9w0BAQsFADAsMSowKAYDVQQDDCFnZW1t
-  YXN0ZXIvREM9amFtZXNyb2JlcnRzb24vREM9ZXUwHhcNMjEwMTIzMTgwNTQ4WhcN
-  MjIwMTIzMTgwNTQ4WjAsMSowKAYDVQQDDCFnZW1tYXN0ZXIvREM9amFtZXNyb2Jl
-  cnRzb24vREM9ZXUwggGiMA0GCSqGSIb3DQEBAQUAA4IBjwAwggGKAoIBgQC3/Li6
-  /b2XI6+wLVA57zxSEu6RtQqu94GlPisH87ziQS+xYDYq8MqrlqHwA2O6GyK+dnCS
-  V0pdXagOSMj27drmy9j6AhdOGxZTvB4M+q2uxxSt1AU8jAlKVBhRiKt15sbSF43K
-  3cY/PrccN4XpJT1YD44WU4+FwaGatFwcAsHtPpxYZgYMZj1sY+wnlKklqBrcWo6H
-  21+fRAVgHi5dqypH6Yez5Rc2rUKEg72OIU76iSjyx9XP8CiX4sxSqw+HRwyCkjKx
-  d5Onb6vhlSyWpBwRmPRP1bbiztxho4Pla28HM+qVvLIqHpCYoWZi2xmKBoJ5oKPx
-  cUj1ThI5nucnixrbg5dUjhJF9Ews4PukHtG2uIc/7WG+vCbZHn2CZNqH4HfB8cW5
-  yoeD4sBFEJwzK8nfGZuAwJ0yOi8yZTqcW8HyK/IVXH+fqC1NsQVEL5yYhnaCCVvx
-  ShG2g5rh48kNN5CsIQb87WP3J/YnsXhIz3+dwJPmLt+QuWE6sI1NkGWHQFECAwEA
-  AaOBijCBhzAJBgNVHRMEAjAAMAsGA1UdDwQEAwIEsDAdBgNVHQ4EFgQUEmUqvmku
-  azMk41rHySinmAUmae8wJgYDVR0RBB8wHYEbZ2VtbWFzdGVyQGphbWVzcm9iZXJ0
+  YXN0ZXIvREM9amFtZXNyb2JlcnRzb24vREM9ZXUwHhcNMjIwMjEwMTcwNzMwWhcN
+  MjMwMjEwMTcwNzMwWjAsMSowKAYDVQQDDCFnZW1tYXN0ZXIvREM9amFtZXNyb2Jl
+  cnRzb24vREM9ZXUwggGiMA0GCSqGSIb3DQEBAQUAA4IBjwAwggGKAoIBgQCjKYHi
+  pAa4sq7iRTV7XB75YoxA+OKNh8nqMnSNeOw3lJ3UYW4ulFKOvD+2LddRH1DD3CrP
+  0AYx2MI56Pu/UgtLZkze5pMf6CqLn8MCvonuSjNTj01bDi+fq8xAKu85LwG3xVP3
+  kqMCppRwlf7wtkc8tkC6vjc+GhggwTda8z7quTtiVpy2r3JJUs4QXDTqzfKNC+la
+  dueojDyI3m6mmaGyoT3k3LpTc0Vjg3JeiYsyWxGvK6R0JWqZiv7uwlxtzvuK/v5h
+  1iMhim2hB/6pFZalAddVfzvPCQiRek7nDKxPgvMDze2GIzvrpOapntu18nC1FYnO
+  kAC8vIStaFxmYzZYMB5h8C5q2xhJ3u6uGdyhpMuwvQOkhq4jdvI8SJ04eBhQH0DI
+  ta8OyiPQoqwegXXhBGU35/yZczB6QFswBaw7jTFcya1YKFDvXmLWOl0g6UOFnl66
+  QXnaC/xwRBh48VGALRPCgHuSf0Ag5LP+rv52s6Fbbs5B6syj059gNLO+DKkCAwEA
+  AaOBijCBhzAJBgNVHRMEAjAAMAsGA1UdDwQEAwIEsDAdBgNVHQ4EFgQU/5KH1BIb
+  QxZC8hteME4fAEsveW8wJgYDVR0RBB8wHYEbZ2VtbWFzdGVyQGphbWVzcm9iZXJ0
   c29uLmV1MCYGA1UdEgQfMB2BG2dlbW1hc3RlckBqYW1lc3JvYmVydHNvbi5ldTAN
-  BgkqhkiG9w0BAQsFAAOCAYEAUyL0Sh3Ld6sK0fldg057McECcaOIyB/nKJApz1er
-  ET9qYthBvhpcaNNo19NFuf9W9UyaE6RI9Eb07DvGYyVP/b0yTrIKyzXzSAwFmrek
-  3VUVAhdqOIrIrf16Zpm4NoOGTOst+6sXZ9KQ52DPPwxdLbRCL7HkUFBMIVf2x2LV
-  XflA8IOlSUPH2vmHJnE0yfs7Lzd5+xYpKeOzQSHo0p+SBDzIim3mOZ3ryf4IZQhv
-  wh0fYGJ2iC/w3rPA78Awm6FNPlGCPjgPIz4mliMicRI/sZmFQvn8+2HYWEzDDOO8
-  zYvUB7rkjDGXYqN1Ft3N6EOuNworOFjUlcPrDFCQHf4UgPjD6Z8WbYkI9c7LWtjc
-  HJCyl/PVyS0Srbl8IPn7JsgUCS7S02KOnVzukQ7PTc1wBlAZ4mNb7N9EOp90GP9a
-  YSREEphg3phQsU9LV4Dlc2Id4gB3W+//c1Ek6TFieKoVemKMkbtB1lgqsEpTCfwh
-  oNUg84DlSmZ2z9LKagBVAlDy
+  BgkqhkiG9w0BAQsFAAOCAYEAXGQs4E7LZmG5OckxSpsJVhHaRYXp7ZghKTbn/K6X
+  fEtU8x781gphdo4wNOg8XA4MGAmZTBRi+7x9JE/NmxkC8KtqMSR89ynUtlhRldkF
+  +4uiJNtUSn9Ya3RyfUADF7gJ9OM4WpjXbpXBLtHe+GETjwl3EfDGQ6SNiUCRRaH+
+  DXthsMNrME6QFQ+n2IW7E2nn6ElLACWRUA1lqyRSC6doomKS3BIOmMmZ4aBrtIlB
+  3zBEnu2eXHgd2OkfIiFO+DU2wVF2cs01V41lk575rciiWKGfybmDT3+CQu3kTUHz
+  PkegIP3JOdHPhLsfzgycgbjOaNSEX21SDhPwmLmSng0oNn56N9Po1qFKUG0+XxvA
+  qkFeIsspayVS94HDHTs68XQx4XsZJHfV/H9Sw331V1H7QJkXdbmEDVHXa5D+lI4g
+  m7hEq9GAsvYejQNMhcxlvi+e70d3QiEg+2SVsBHaUGURzv0LjycCOjOaUxQA5swC
+  JKGINad8xoEL7XIDoO6EnBcY
   -----END CERTIFICATE-----
-date: 2021-02-01 00:00:00.000000000 Z
+date: 2022-02-10 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
-  name: dxlite
+  name: c32
   requirement: !ruby/object:Gem::Requirement
     requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.3.0
     - - "~>"
       - !ruby/object:Gem::Version
         version: '0.3'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 0.3.2
+        version: 0.3.0
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.3'
+- !ruby/object:Gem::Dependency
+  name: dxlite
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.5'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.5.1
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.3'
+        version: '0.5'
     - - ">="
       - !ruby/object:Gem::Version
-        version: 0.3.2
+        version: 0.5.1
 description: 
 email: digital.robertson@gmail.com
 executables: []
@@ -76,7 +96,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
-      version: 2.5.3
+      version: 3.0.2
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="