mdqt 0.5.0 → 0.6.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: c8731e9b40b397c638c8bb0e11acc9c954a8aaf940cd558683ff0e14df68a668
-  data.tar.gz: a009f1591877bb571ab8d1f967545dbc836b7f3953e267548433352a122bc449
+  metadata.gz: ecdc970f76e871f3a12cd67229ca3f5a07888f0f633a21e0aead8a0a235b5f75
+  data.tar.gz: 9576e8403a2d3c6f755a9007a341f73c9e44a18ca0d96d869c1867eff00c9855
 SHA512:
-  metadata.gz: e802b9620268525cf5b4249c59e604ba5345e45805b15e9c57ed4daee38271712f4d8f9bc8213a2883c9ac39f8a303bbeaec1dd8eacc09f223a61aa4d6ae5b07
-  data.tar.gz: b37ed2ddf5e7a9212e4ab5f722a77b8c2fe0bde7e6e10c74203f7fe2b7c9863f873eb491e5fea0d121b2281e6955122d9f0d21c3d9d201c4c82c8743aa952067
+  metadata.gz: 0b01525dbee9c0aa0c6f7a1ec46d21bc101ac47e5a70e2e7a5b0442d5efb10477852dac8389569cfea8ddeb1ec642eea2ad011a2d9988be1dd215d5c47c75e2f
+  data.tar.gz: 935e467dc9dfe24c06646b30527ad035e4901943d7393677b72aa6df623ad161f109083e921e5d64ca63bcca2cb25e75dcdd0b5a618a7ef66837dafbf799f287

data/CHANGELOG.md

@@ -1,5 +1,30 @@
 # Changelog
 
+## 0.6.0
+
+### New Features
+
+- STDIN and pipes: Arguments (such as filenames and entity IDs) can now be piped into mdqt. This 
+  enables pipelining, so you can chain commands together.
+- The `rename` command now has a `--link` option that creates a symlink from the original
+  filename to the renamed file.
+- The `get` command now has a `--list` option that works when `--save-to` is used, to list filenames being
+  written to disk. 
+
+### Improvements
+
+- Emacs backup files (so called turd files) ending with ~ and files ending with .bak 
+  are now ignored.
+
+### Removed features
+
+- `link` and `rename` now require files to be specified: you now *cannot* run `mdqt rename`
+  to rename everything in the current directory.
+
+- The `--link_id` option for `get` saved a link to each downloaded file that is almost the same as the 
+  filename - maybe this made sense in mdqt 0.1.0 but it's quite useless now. If anyone can remember what it was
+  actually for I'll put it back.
+
 ## 0.5.0
 
 ### New Features

data/README.md

@@ -246,6 +246,16 @@ To show the full MDQ URL of an entity
 
     $ mdqt url http://entity.ac.uk/shibboleth
 
+MDQT can accept input on stdin, allowing composition and pipelining
+
+    $ cat list_of_ids.txt | bundle exec mdqt url
+
+    $ mdqt list | grep cern.ch | mdqt get --save-to cern_metadata/ --list  | mdqt ls
+
+## Alternatives
+
+  * [SAML Library](https://github.com/trscavo/saml-library) is a set of scripts to help with metadata-related tasks, written
+by Tom Scavo of Internet2. Some of the scripts provide similar functionality to MDQT, and are designed to be piped together.
 
 ## Library Usage
 

data/exe/mdqt

@@ -42,7 +42,8 @@ Commander.configure do
     c.option '--explain', 'Show details of client request and server response'
     c.option '--tls-risky', "Don't check certificate used for TLS (usually a bad idea)"
     c.option '--save-to PATH', String, 'Write all data to files in the specified directory'
-    c.option '--link-id', 'If saving files, save files with aliases (requires `--save-to`)'
+    #c.option '--link-id', 'If saving files, save files with aliases (requires `--save-to`)'
+    c.option '--list', 'If saving files, print the names of files written to disk (requires `--save-to`)'
     c.option '--verbose', 'Display extra information on stderr'
     c.action do |args, options|
       options.default MDQT::CLI::Defaults.cli_defaults
@@ -103,7 +104,6 @@ Commander.configure do
     c.option '--force', 'Overwrite any existing links'
     c.option '--verbose', 'Display extra information on stderr'
     c.action do |args, options|
-      args = Dir.glob("*.xml") unless args && !args.empty?
       options.default MDQT::CLI::Defaults.cli_defaults
       options.default({ service: :not_required })
       MDQT::CLI::Ln.run(args, options)
@@ -115,7 +115,6 @@ Commander.configure do
     c.description = 'List valid metadata files in directory'
     c.option '--verbose', 'Display extra information on stderr'
     c.action do |args, options|
-      args = Dir.glob("*.xml") unless args && !args.empty?
       options.default MDQT::CLI::Defaults.cli_defaults
       options.default({ service: :not_required })
       MDQT::CLI::Ls.run(args, options)
@@ -128,7 +127,6 @@ Commander.configure do
     c.option '--service URL', String, 'MDQ service to search for entities. Defaults to MDQT_SERVICE or MDQ_BASE_URL env variables'
     c.option '--cache', "Cache downloads and try to fetch from cache where appropriate (deprecated)"
     c.option '--refresh', "Never cache (will prevent --cache)"
-    #c.option '--stdin', 'accept one or more entity ids from STDIN'
     c.option '--verbose', 'Display extra information on stderr'
     c.action do |args, options|
       options.default MDQT::CLI::Defaults.cli_defaults
@@ -152,8 +150,8 @@ Commander.configure do
     c.description = 'Rename a file using the sha1 hash of its entityID'
     c.option '--force', 'Overwrite any existing files with that name'
     c.option '--verbose', 'Display extra information on stderr'
+    c.option '--link', 'Add a symlink with the original filename'
     c.action do |args, options|
-      args = Dir.glob("*.xml") unless args && !args.empty?
       options.default MDQT::CLI::Defaults.cli_defaults
       options.default({ service: :not_required })
       MDQT::CLI::Rename.run(args, options)

data/lib/mdqt/cli/base.rb

@@ -6,19 +6,18 @@ module MDQT
 
       require 'mdqt/cli'
       require 'pastel'
+      require 'pathname'
 
       def self.run(args, options)
 
-        #args = options.stdin ? absorb_piped_args(args) : args
-
-        check_requirements(args, options)
-        introduce(args, options)
+        check_requirements(options)
+        introduce(options)
 
         self.new(args, options).run
       end
 
-      def self.check_requirements(args, options)
-        
+      def self.check_requirements(options)
+
         unless options.service == :not_required
           abort "No MDQ service URL has been specified. Please use --service, MDQT_SERVICE or MDQ_BASE_URL" unless service_url(options).to_s.start_with?("http")
         end
@@ -30,47 +29,41 @@ module MDQT
           rescue
             abort "Error: Directory #{dir} did not exist, and we can't create it"
           end
-          abort "Error: '#{dir}' is not a writable directory!" if (File.directory?(dir) && ! File.writable?(dir))
+          abort "Error: '#{dir}' is not a writable directory!" if (File.directory?(dir) && !File.writable?(dir))
           abort "Error: '#{dir}' is not a directory!" unless File.directory?(dir)
         end
 
       end
 
-      def self.introduce(args, options)
+      def self.introduce(options)
         if options.verbose
           STDERR.puts "MDQT version #{MDQT::VERSION}"
           STDERR.puts "Using #{service_url(options)}" unless options.service == :not_required
           STDERR.puts "Caching is #{MDQT::CLI::CacheControl.caching_on?(options) ? 'on' : 'off'}"
           STDERR.print "XML validation is #{MDQT::Client.verification_available? ? 'available' : 'not available'}"
-          STDERR.puts  " #{options.validate ? "and active" : "but inactive"} for this request" if MDQT::Client.verification_available?
+          STDERR.puts " #{options.validate ? "and active" : "but inactive"} for this request" if MDQT::Client.verification_available?
           STDERR.print "Signature verification is #{MDQT::Client.verification_available? ? 'available' : 'not available'}"
-          STDERR.puts  " #{options.verify_with ? "and active" : "but inactive"} for this request" if MDQT::Client.verification_available?
+          STDERR.puts " #{options.verify_with ? "and active" : "but inactive"} for this request" if MDQT::Client.verification_available?
           STDERR.puts "Output directory for saved files is: #{File.absolute_path(options.save_to)}" if options.save_to
           STDERR.puts("Warning! TLS certificate verification has been disabled!") if options.tls_risky
           STDERR.puts
         end
       end
 
-      def self.absorb_piped_args(args)
-        piped = piped? ? parse_stdin : []
-        args + piped
-      end
-
-      def self.piped?
-        !STDIN.tty? && !STDIN.closed?
-      end
-
-      def self.parse_stdin
-        STDIN.gsub("\s", "\n").each_line.collect {|l| l.strip}
+      def initialize(cli_args, options)
+        piped_input = get_stdin
+        @args = cli_args.concat(piped_input)
+        @options = options
       end
 
-      def initialize(args, options)
-        @args = args
-        @options = options
+      def get_stdin
+        return $stdin.readlines.map(&:split).flatten.map(&:strip) if pipeable?
+        []
       end
 
-      def args=(new_args)
-        @args = new_args
+      def pipeable?
+        return false if ENV["MDQT_STDIN"].to_s.strip.downcase == "off" # Workaround Aruba testing weirdness?
+        !STDIN.tty? && !$stdin.closed? && $stdin.stat.pipe?
       end
 
       def args
@@ -119,7 +112,7 @@ module MDQT
         unless response.explanation.empty?
           require 'terminal-table'
           misc_rows = [['URL', response.explanation[:url]], ["Method", response.explanation[:method]], ['Status', response.explanation[:status]]]
-          req_header_rows  = response.explanation[:request_headers].map { |k, v| ['C', k, v] }
+          req_header_rows = response.explanation[:request_headers].map { |k, v| ['C', k, v] }
           resp_header_rows = response.explanation[:response_headers].map { |k, v| ['S', k, v] }
 
           btw Terminal::Table.new :title => "HTTP Misc", :rows => misc_rows
@@ -182,6 +175,7 @@ module MDQT
     end
 
   end
-#
+
+  #
 end
 

data/lib/mdqt/cli/get.rb

@@ -84,22 +84,28 @@ module MDQT
       end
 
       def output_files(results, options)
+        pwd = Pathname.getwd
         prepare_output_directory(options.save_to)
         results.each do |result|
           main_file = output_file_path(result.filename)
           open(main_file, 'w') { |f|
             f.puts result.data
           }
-          yay "Created #{main_file}"
 
-          if options.link_id
-            ["{sha1}#{result.filename.gsub(".xml", "")}"].each do |altname|
-              full_alias = output_file_path(altname)
-              FileUtils.ln_sf(main_file, full_alias)
-              yay "Linked alias #{altname} -> #{main_file}"
-            end
+          if options.list
+            puts Pathname.new(main_file).relative_path_from(pwd)
           end
 
+          yay "Created #{main_file}"
+
+          # if options.link_id
+          #   ["{sha1}#{result.filename.gsub(".xml", "")}"].each do |altname|
+          #     full_alias = output_file_path(altname)
+          #     FileUtils.ln_sf(main_file, full_alias)
+          #     yay "Linked alias #{altname} -> #{main_file}"
+          #   end
+          # end
+
         end
       end
 

data/lib/mdqt/cli/ln.rb

@@ -19,6 +19,8 @@ module MDQT
           explain: options.explain ? true : false,
         )
 
+        halt!("Please specify a file to link to!") if args.empty?
+
         args.each do |filename|
 
           next if File.symlink?(filename)
@@ -44,6 +46,11 @@ module MDQT
             btw("Cannot link file to itself! #{filename}")
           end
 
+          if file.turd?
+            hey "Warning: will not process backup/turd files"
+            next
+          end
+
           message = ""
 
           if File.exists?(linkname)

data/lib/mdqt/cli/ls.rb

@@ -21,7 +21,9 @@ module MDQT
 
         results = []
 
-        args.each do |filename|
+        p_args = args.empty? ? Dir.glob("*.xml") : args
+
+        p_args.each do |filename|
 
           file = client.open_metadata(filename)
 

data/lib/mdqt/cli/rename.rb

@@ -19,8 +19,12 @@ module MDQT
           explain: options.explain ? true : false,
         )
 
+        halt!("Please specify a file to rename!") if args.empty?
+
         args.each do |filename|
 
+          next if File.symlink?(filename)
+
           file = client.open_metadata(filename)
 
           halt!("Cannot access file #{filename}") unless file.readable?
@@ -32,6 +36,11 @@ module MDQT
           newname = file.linkname # Using the same name as the link, not super-obvious
           next if filename == newname
 
+          if file.turd?
+            hey "Warning: will not process backup/turd files"
+            next
+          end
+
           message = ""
 
           if File.exists?(newname)
@@ -44,6 +53,12 @@ module MDQT
           end
 
           File.rename(filename, newname)
+
+          if options.link
+            File.delete(filename) if options.force && File.exists?(filename)
+            File.symlink(newname, filename) unless newname == filename
+          end
+
           hey("#{filename} renamed to #{newname} [#{file.entity_id}] #{message}") if options.verbose
         end
 

data/lib/mdqt/client/metadata_file.rb

@@ -43,6 +43,12 @@ module MDQT
         File.readable?(filename)
       end
 
+      def turd?
+        return true if basename.end_with?("~")
+        return true if basename.end_with?(".bak")
+        false
+      end
+
       def type
         @type ||= calculate_type
       end

data/lib/mdqt/version.rb

@@ -1,3 +1,3 @@
 module MDQT
-  VERSION = "0.5.0"
+  VERSION = "0.6.0"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: mdqt
 version: !ruby/object:Gem::Version
-  version: 0.5.0
+  version: 0.6.0
 platform: ruby
 authors:
 - Pete Birkinshaw
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2021-12-23 00:00:00.000000000 Z
+date: 2022-04-14 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: commander
@@ -292,7 +292,6 @@ files:
 - lib/mdqt/cli/rename.rb
 - lib/mdqt/cli/reset.rb
 - lib/mdqt/cli/services.rb
-- lib/mdqt/cli/stream.rb
 - lib/mdqt/cli/transform.rb
 - lib/mdqt/cli/url.rb
 - lib/mdqt/cli/version.rb