dtas 0.15.0 → 0.16.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
-SHA1:
-  metadata.gz: b38ad68d0ac0b303e95ef7a6de996581fd4a5c73
-  data.tar.gz: 84066e2d52ede911d045b0d9f62a6d0cb741a0a3
+SHA256:
+  metadata.gz: d887afd41dc71abd3448e72c3d33d188e4629b8dbc576af3237c3bd47b246581
+  data.tar.gz: cf5262f4c45becd00e75dc2f1c3de5ab05b27bd9a029c9fef458faec5cc58303
 SHA512:
-  metadata.gz: e02cd6429e8df6fca2d6d0745e5fa1094910c6688ecbbc404d309f044a6abba32dfc0f0b03f60f5d8a487f94fc64fe6cedc0365c5a5cf539fb6d1db80b05bcad
-  data.tar.gz: 8849acee6e2ec2ec588f686d7d56e741db25e480a8d9661d0feee4f95fe6f8dbd030ae07806654b7aa6020abd46243bfab8a7f33e25e977467aafe1b7a824352
+  metadata.gz: 94f470ad51f9723bf5d3161b445d0434e6e5e45349fc6c5227ecb80aa0c9d1409357979cd8f97edc909bed6c65b75494757b731d2f7cd97f4ed345640465aa7a
+  data.tar.gz: 631dc0a9b16771662bb6f8215ab9ee71acd078ac7cae26f1b90408106da840909d59f99cdf569425b2be05f2c9018f032ad426405ca1bbe1df28af005d2f020c

data/GIT-VERSION-GEN

@@ -1,11 +1,11 @@
 #!/usr/bin/env ruby
-# Copyright (C) 2013-2016 all contributors <dtas-all@nongnu.org>
+# Copyright (C) 2013-2019 all contributors <dtas-all@nongnu.org>
 # License: GPL-3.0+ <https://www.gnu.org/licenses/gpl-3.0.txt>
 # frozen_string_literal: true
 CONSTANT = "DTAS::VERSION"
 RVF = "lib/dtas/version.rb"
 GVF = "GIT-VERSION-FILE"
-DEF_VER = "v0.15.0"
+DEF_VER = "v0.16.0"
 vn = DEF_VER
 
 # First see if there is a version file (included in release tarballs),

data/INSTALL

@@ -19,13 +19,13 @@ Debian 7+ users can install dependencies easily:
 # installing dtas RubyGem on GNU/Linux (Linux kernel 2.6.32+)
 
 Be sure to have Ruby development headers and a working C compiler.
-This will pull in the io_splice and sleepy_penguin RubyGems for minor
+This will pull in the sleepy_penguin RubyGems for minor
 speedups.  If you cannot be bothered to have a development
 environment, just use "gem install dtas".
 
     sudo gem install dtas-linux
 
-This should pull in the "io_splice" and "sleepy_penguin" RubyGems
+This should pull in the "sleepy_penguin" RubyGems
 
 For future upgrades of dtas (upgrades to dtas-linux will be infrequent)
 
@@ -39,16 +39,14 @@ For future upgrades of dtas (upgrades to dtas-linux will be infrequent)
 
 Grab the latest tarball from our HTTPS site:
 
-    https://80x24.org/dtas/2017/dtas-0.15.0.tar.gz
+    https://80x24.org/dtas/2019/dtas-0.16.0.tar.gz
 
-    $ tar zxvf dtas-0.15.0.tar.gz
-    $ cd dtas-0.15.0
+    $ tar zxvf dtas-0.16.0.tar.gz
+    $ cd dtas-0.16.0
     $ sudo ruby setup.rb
 
-GNU/Linux users may optionally install "io_splice" and
-"sleepy_penguin" packages:
+GNU/Linux users may optionally install the "sleepy_penguin" package:
 
-    * io_splice - https://bogomips.org/ruby_io_splice/
     * sleepy_penguin - https://bogomips.org/sleepy_penguin/
 
 # CONTACT
@@ -61,5 +59,5 @@ No subscription is necessary to post to the mailing list.
 
 # COPYRIGHT
 
-Copyright 2013-2016 all contributors <dtas-all@nongnu.org>
+Copyright 2013-2019 all contributors <dtas-all@nongnu.org>
 License: GPL-3.0+ <https://www.gnu.org/licenses/gpl-3.0.txt>

data/README

@@ -59,7 +59,7 @@ Coming:
 
 ## Source code
 
-  git clone git://80x24.org/dtas
+  git clone https://80x24.org/dtas.git
 
 Please use git-format-patch(1) and git-send-email(1) distributed with
 the git(7) suite for generating and sending patches.  Please format
@@ -86,7 +86,7 @@ Atom: <https://80x24.org/dtas-all/new.atom>
 
 ## Copyright
 
-Copyright 2013-2016 all contributors <dtas-all@nongnu.org>\
+Copyright 2013-2019 all contributors <dtas-all@nongnu.org>\
 License: GPL-3.0+ <https://www.gnu.org/licenses/gpl-3.0.txt>
 
 dtas is copyrighted Free Software by all contributors, see logs

data/Rakefile

@@ -1,4 +1,4 @@
-# Copyright (C) 2013-2016 all contributors <dtas-all@nongnu.org>.
+# Copyright (C) 2013-2019 all contributors <dtas-all@nongnu.org>.
 # License: GPL-3.0+ <https://www.gnu.org/licenses/gpl-3.0.txt>
 # frozen_string_literal: true
 require 'tempfile'
@@ -93,7 +93,7 @@ def tags
 end
 
 task rsync_docs: %w(NEWS NEWS.atom) do
-  dest = ENV["RSYNC_DEST"] || "80x24.org:/srv/dtas/"
+  dest = ENV["RSYNC_DEST"] || "80x24.org:/srv/80x24/dtas/"
   top = %w(INSTALL NEWS README COPYING NEWS.atom)
   files = []
 

data/bin/dtas-console

@@ -1,5 +1,5 @@
 #!/usr/bin/env ruby
-# Copyright (C) 2013-2016 all contributors <dtas-all@nongnu.org>
+# Copyright (C) 2013-2019 all contributors <dtas-all@nongnu.org>
 # License: GPL-3.0+ <https://www.gnu.org/licenses/gpl-3.0.txt>
 # frozen_string_literal: true
 #
@@ -173,11 +173,11 @@ def may_fail(res, events)
       Curses.setpos(lineno += 1, 0)
       Curses.clrtoeol
       Curses.addstr(infile)
-      total = " [#{Time.at(samples / rate).strftime(tfmt)}]" if samples != 0
+      total = " [#{Time.at(samples / rate).utc.strftime(tfmt)}]" if samples != 0
       Curses.setpos(lineno += 1, 0)
       Curses.clrtoeol
       if rate != 0
-        Curses.addstr("#{Time.at(elapsed).strftime(tfmt)}#{total} #{fmt}")
+        Curses.addstr("#{Time.at(elapsed).utc.strftime(tfmt)}#{total} #{fmt}")
       else
         Curses.addstr("#{elapsed} samples #{total} #{fmt}")
       end

data/bin/dtas-readahead

@@ -1,5 +1,5 @@
 #!/usr/bin/env ruby
-# Copyright (C) 2015-2016 all contributors <dtas-all@nongnu.org>
+# Copyright (C) 2015-2019 all contributors <dtas-all@nongnu.org>
 # License: GPL-3.0+ <https://www.gnu.org/licenses/gpl-3.0.txt>
 # frozen_string_literal: true
 #
@@ -75,7 +75,7 @@ def seek_to_cur_pos(cur_pid, fp)
 end
 
 def children_of(ppid)
-  `ps h -o pid --ppid=#{ppid}`.split(/\s+/s).map(&:to_i)
+  `ps h -o pid --ppid=#{ppid}`.split(/\s+/).map(&:to_i)
 end
 
 def expand_pid(pid)

data/dtas-linux.gemspec

@@ -1,4 +1,4 @@
-# Copyright (C) 2013-2016 all contributors <dtas-all@nongnu.org>
+# Copyright (C) 2013-2019 all contributors <dtas-all@nongnu.org>
 # License: GPL-3.0+ <https://www.gnu.org/licenses/gpl-3.0.txt>
 #
 # this just declares dependencies to make gem installation a little easier
@@ -13,8 +13,7 @@
   s.email = %q{e@80x24.org}
   s.files = []
   s.homepage = 'https://80x24.org/dtas/'
-  s.add_dependency(%q<dtas>)
-  s.add_dependency(%q<io_splice>, '~> 4')
-  s.add_dependency(%q<sleepy_penguin>, '~> 3')
+  s.add_dependency(%q<dtas>, '~> 0.16')
+  s.add_dependency(%q<sleepy_penguin>, '~> 3.5')
   s.licenses = 'GPL-3.0+'
 end

data/lib/dtas.rb

@@ -1,4 +1,4 @@
-# Copyright (C) 2013-2016 all contributors <dtas-all@nongnu.org>
+# Copyright (C) 2013-2019 all contributors <dtas-all@nongnu.org>
 # License: GPL-3.0+ <https://www.gnu.org/licenses/gpl-3.0.txt>
 # frozen_string_literal: true
 
@@ -42,3 +42,5 @@ def self.dedupe_str(str)
 
 require_relative 'dtas/compat_onenine'
 require_relative 'dtas/spawn_fix'
+require_relative 'dtas/encoding'
+DTAS.extend(DTAS::Encoding)

data/lib/dtas/buffer.rb

@@ -1,4 +1,4 @@
-# Copyright (C) 2013-2016 all contributors <dtas-all@nongnu.org>
+# Copyright (C) 2013-2019 all contributors <dtas-all@nongnu.org>
 # License: GPL-3.0+ <https://www.gnu.org/licenses/gpl-3.0.txt>
 # frozen_string_literal: true
 require 'io/wait'
@@ -8,7 +8,9 @@
 class DTAS::Buffer # :nodoc:
   begin
     raise LoadError, "no splice with _DTAS_POSIX" if ENV["_DTAS_POSIX"]
-    require 'io/splice' # splice is only in Linux for now...
+    require 'sleepy_penguin' # splice is only in Linux for now...
+    SleepyPenguin.respond_to?(:splice) or
+      raise LoadError, 'sleepy_penguin 3.5+ required for splice', []
     require_relative 'buffer/splice'
     include DTAS::Buffer::Splice
   rescue LoadError

data/lib/dtas/buffer/read_write.rb

@@ -1,4 +1,4 @@
-# Copyright (C) 2013-2016 all contributors <dtas-all@nongnu.org>
+# Copyright (C) 2013-2019 all contributors <dtas-all@nongnu.org>
 # License: GPL-3.0+ <https://www.gnu.org/licenses/gpl-3.0.txt>
 # frozen_string_literal: true
 require 'io/nonblock'
@@ -7,7 +7,7 @@
 require_relative '../nonblock'
 
 # compatibility code for systems lacking "splice" support via the
-# "io-splice" RubyGem.  Used only by -player
+# "sleepy_penguin" 3.5+ RubyGem.  Used only by -player
 module DTAS::Buffer::ReadWrite # :nodoc:
   MAX_AT_ONCE = 512 # min PIPE_BUF value in POSIX
   attr_accessor :buffer_size

data/lib/dtas/buffer/splice.rb

@@ -1,16 +1,18 @@
-# Copyright (C) 2013-2016 all contributors <dtas-all@nongnu.org>
+# Copyright (C) 2013-2019 all contributors <dtas-all@nongnu.org>
 # License: GPL-3.0+ <https://www.gnu.org/licenses/gpl-3.0.txt>
 # frozen_string_literal: true
 require 'io/nonblock'
-require 'io/splice'
+require 'sleepy_penguin'
 require_relative '../../dtas'
 require_relative '../pipe'
 
-# Used by -player on Linux systems with the "io-splice" RubyGem installed
+# Used by -player on Linux systems with the "sleepy_penguin" RubyGem installed
 module DTAS::Buffer::Splice # :nodoc:
   MAX_AT_ONCE = 4096 # page size in Linux
   MAX_AT_ONCE_1 = 65536
-  F_MOVE = IO::Splice::F_MOVE
+  F_MOVE = SleepyPenguin::F_MOVE
+  F_NONBLOCK = SleepyPenguin::F_NONBLOCK
+  TRY = { exception: false }.freeze
 
   def buffer_size
     @to_io.pipe_size
@@ -24,13 +26,13 @@ def buffer_size=(bytes)
 
   # be sure to only call this with nil when all writers to @wr are done
   def discard(bytes)
-    IO.splice(@to_io, nil, DTAS.null, nil, bytes)
+    SleepyPenguin.splice(@to_io, DTAS.null, bytes)
   end
 
   def broadcast_one(targets, limit = nil)
     # single output is always non-blocking
     limit ||= MAX_AT_ONCE_1
-    s = IO.trysplice(@to_io, nil, targets[0], nil, limit, F_MOVE)
+    s = SleepyPenguin.splice(@to_io, targets[0], limit, F_MOVE, TRY)
     if Symbol === s
       targets # our one and only target blocked on write
     else
@@ -46,7 +48,7 @@ def broadcast_one(targets, limit = nil)
   def __tee_in_full(src, dst, bytes)
     rv = 0
     while bytes > 0
-      s = IO.tee(src, dst, bytes)
+      s = SleepyPenguin.tee(src, dst, bytes)
       bytes -= s
       rv += s
     end
@@ -56,7 +58,7 @@ def __tee_in_full(src, dst, bytes)
   def __splice_in_full(src, dst, bytes, flags)
     rv = 0
     while bytes > 0
-      s = IO.splice(src, nil, dst, nil, bytes, flags)
+      s = SleepyPenguin.splice(src, dst, bytes, flags)
       rv += s
       bytes -= s
     end
@@ -69,7 +71,7 @@ def __broadcast_tee(blocked, targets, chunk_size)
     targets.delete_if do |dst|
       begin
         t = (dst.nonblock? || most_teed == 0) ?
-            IO.trytee(@to_io, dst, chunk_size) :
+            SleepyPenguin.tee(@to_io, dst, chunk_size, F_NONBLOCK, TRY) :
             __tee_in_full(@to_io, dst, chunk_size)
         if Integer === t
           if t > most_teed
@@ -117,7 +119,7 @@ def broadcast_inf(targets, limit = nil)
     begin
       targets << last
       if last.nonblock? || most_teed == 0
-        s = IO.trysplice(@to_io, nil, last, nil, bytes, F_MOVE)
+        s = SleepyPenguin.splice(@to_io, last, bytes, F_MOVE|F_NONBLOCK, TRY)
         if Symbol === s
           blocked << last
 

data/lib/dtas/encoding.rb

@@ -0,0 +1,58 @@
+# Copyright (C) 2018-2019 all contributors <dtas-all@nongnu.org>
+# License: GPL-3.0+ <https://www.gnu.org/licenses/gpl-3.0.txt>
+# frozen_string_literal: true
+
+# This module gets included in DTAS
+module DTAS::Encoding # :nodoc:
+  def self.extended(mod)
+    mod.instance_eval { @charlock_holmes = nil}
+  end
+
+private
+
+  def try_enc_harder(str, enc, old) # :nodoc:
+    case @charlock_holmes
+    when nil
+      begin
+        require 'charlock_holmes'
+        @charlock_holmes = CharlockHolmes::EncodingDetector.new
+      rescue LoadError
+        warn "`charlock_holmes` gem not available for encoding detection"
+        @charlock_holmes = false
+      end
+    when false
+      enc_fallback(str, enc, old)
+    else
+      res = @charlock_holmes.detect(str)
+      if det = res[:ruby_encoding]
+        str.force_encoding(det)
+        warn "charlock_holmes detected #{str.inspect} as #{det}..."
+        str.valid_encoding? or enc_fallback(str, det, old)
+      else
+        enc_fallback(str, enc, old)
+      end
+    end
+    str
+  end
+
+  def enc_fallback(str, enc, old) # :nodoc:
+    str.force_encoding(old)
+    warn "could not detect encoding for #{str.inspect} (not #{enc})"
+  end
+
+public
+
+  def try_enc(str, enc, harder = true) # :nodoc:
+    old = str.encoding
+    return str if old == enc
+    str.force_encoding(enc)
+    unless str.valid_encoding?
+      if harder
+        try_enc_harder(str, enc, old)
+      else
+        enc_fallback(str, enc, old)
+      end
+    end
+    str
+  end
+end

data/lib/dtas/mlib.rb

@@ -1,5 +1,5 @@
 # -*- encoding: utf-8 -*-
-# Copyright (C) 2015-2016 all contributors <dtas-all@nongnu.org>
+# Copyright (C) 2015-2019 all contributors <dtas-all@nongnu.org>
 # License: GPL-3.0+ <https://www.gnu.org/licenses/gpl-3.0.txt>
 # frozen_string_literal: true
 #
@@ -10,6 +10,7 @@
 require_relative 'source/ff'
 require_relative 'source/splitfx'
 require 'socket'
+require 'tempfile'
 
 # For the DTAS Music Library, based on what MPD uses.
 class DTAS::Mlib # :nodoc:
@@ -37,16 +38,13 @@ class DTAS::Mlib # :nodoc:
 
   def initialize(db)
     if String === db
+      require 'sequel'
+      opts = { single_threaded: true }
       db = "sqlite://#{db}" unless db.include?('://')
-      require 'sequel/no_core_ext'
-      db = Sequel.connect(db, single_threaded: true)
-    end
-    if db.class.to_s.downcase.include?('sqlite')
-      db.transaction_mode = :immediate
-      db.synchronous = :off
-      db.case_sensitive_like = false
-    else
-      warn 'non-SQLite databases may not work in the future'
+      opts[:transaction_mode] = :immediate
+      opts[:synchronous] = :off
+      opts[:case_sensitive_like] = false # only for 'search'
+      db = Sequel.connect(db, opts)
     end
     @db = db
     @pwd = nil
@@ -65,15 +63,23 @@ def initialize(db)
     ]
   end
 
+  def synchronize
+    @lock.flock(File::LOCK_EX)
+    @db.transaction { yield }
+  ensure
+    @lock.flock(File::LOCK_UN)
+  end
+
   def init_suffixes
-    `sox --help 2>/dev/null` =~ /\nAUDIO FILE FORMATS:\s*([^\n]+)/s
+    `sox --help 2>/dev/null` =~ /\nAUDIO FILE FORMATS:\s*([^\n]+)/
     re = $1.split(/\s+/).map! { |x| Regexp.quote(x) }.join('|')
     @suffixes = Regexp.new("\\.(?:#{re}|yml)\\z", Regexp::IGNORECASE)
   end
 
-  def worker(todo)
+  def worker(todo, lock)
+    @lock = lock
     @work.close
-    @db.tables # reconnect before chdir
+    synchronize { @db.tables } # reconnect before chdir
     @pwd = Dir.pwd.b
     begin
       buf = todo.recv(16384) # 4x bigger than PATH_MAX ought to be enough
@@ -87,7 +93,7 @@ def worker(todo)
   end
 
   def ignore(job)
-    @db.transaction do
+    synchronize do
       node_ensure(job.parent_id, job.path, DM_IGN, job.ctime)
     end
   end
@@ -106,24 +112,16 @@ def worker_work(job)
     end
     return ignore(job) unless found
     tlen = found.duration
-    return ignore(job) if tlen < 0
+    return ignore(job) if tlen.nil? || tlen < 0
     tlen = tlen.round
     tmp = {}
-    found.comments.each do |tag, value|
-      tag_id = @tag_map[tag] or next
-      value.strip!
-
-      # FIXME: this fallback needs testing
-      [ Encoding::UTF_8, Encoding::ISO_8859_1 ].each do |enc|
-        value.force_encoding(enc)
-        if value.valid_encoding?
-          value.encode!(Encoding::UTF_8) if enc != Encoding::UTF_8
-          tmp[tag_id] = value
-          break
-        end
+    if comments = found.comments
+      comments.each do |tag, value|
+        tag_id = @tag_map[tag] or next
+        tmp[tag_id] = value if value.valid_encoding?
       end
     end
-    @db.transaction do
+    synchronize do
       node_id = node_ensure(job.parent_id, path, tlen, job.ctime)[:id]
       vals = @db[:vals]
       comments = @db[:comments]
@@ -154,11 +152,20 @@ def update(path, opts = nil)
     @work and raise 'update already running'
     todo, @work = UNIXSocket.pair(:SOCK_SEQPACKET)
     @db.disconnect
-    jobs.times { |i| fork { worker(todo) } }
+
+    # like a Mutex between processes
+    @lock = Tempfile.new('dtas.mlib.lock')
+    jobs.times do |i|
+      lock = File.open(@lock.path, 'w')
+      fork { worker(todo, lock) }
+      lock.close
+    end
+    @lock.unlink
     todo.close
     scan_dir(path, st)
     @work.close
     Process.waitall
+    @lock.close
   ensure
     @work = nil
   end
@@ -246,7 +253,7 @@ def root_node
   end
 
   def dir_vivify(parts, ctime)
-    @db.transaction do
+    synchronize do
       dir = root_node
       last = parts.pop
       parts.each do |name|
@@ -300,7 +307,7 @@ def scan_dir(path, st, parent_id = nil)
       dir = dir_vivify(@pwd.split(%r{/+}n), st.ctime.to_i)
       dir_id = dir[:id]
 
-      @db.transaction do
+      synchronize do
         @db[:nodes].where(parent_id: dir_id).each do |node|
           File.exist?(node[:name]) or remove_entry(node)
         end

data/lib/dtas/pipe.rb

@@ -1,8 +1,8 @@
-# Copyright (C) 2013-2016 all contributors <dtas-all@nongnu.org>
+# Copyright (C) 2013-2019 all contributors <dtas-all@nongnu.org>
 # License: GPL-3.0+ <https://www.gnu.org/licenses/gpl-3.0.txt>
 # frozen_string_literal: true
 begin
-  require 'io/splice'
+  require 'sleepy_penguin'
 rescue LoadError
 end
 require_relative '../dtas'
@@ -20,12 +20,15 @@ def self.new
     rv
   end
 
-  # create no-op methods for non-Linux
-  unless method_defined?(:pipe_size=)
-    def pipe_size=(_)
-    end
+  def pipe_size=(nr)
+    defined?(SleepyPenguin::F_SETPIPE_SZ) and
+      fcntl(SleepyPenguin::F_SETPIPE_SZ, nr)
   end
 
+  def pipe_size
+    fcntl(SleepyPenguin::F_GETPIPE_SZ)
+  end if defined?(SleepyPenguin::F_GETPIPE_SZ)
+
   # avoid syscall, we never change IO#nonblock= directly
   def nonblock?
     false

data/lib/dtas/pipeline.rb

@@ -0,0 +1,75 @@
+# Copyright (C) 2017-2019 all contributors <dtas-all@nongnu.org>
+# License: GPL-3.0+ <https://www.gnu.org/licenses/gpl-3.0.txt>
+# frozen_string_literal: true
+require_relative '../dtas'
+require_relative 'spawn_fix'
+
+module DTAS::Pipeline # :nodoc:
+  include DTAS::SpawnFix
+
+  # Process.spawn wrapper which supports running Proc-like objects in
+  # a separate process, not just external commands.
+  # Returns the pid of the spawned process
+  def pspawn(env, cmd, rdr = {})
+    case cmd
+    when Array
+      spawn(env, *cmd, rdr)
+    else # support running Proc-like objects, too:
+      fork do
+        ENV.update(env) if env
+
+        # setup redirects
+        [ $stdin, $stdout, $stderr ].each_with_index do |io, fd|
+          dst = rdr[fd] and io.reopen(dst)
+        end
+
+        # close all other pipes, since we can't rely on FD_CLOEXEC
+        # (as we do not exec, here)
+        rdr.each do |k, v|
+          k.close if v == :close
+        end
+        cmd.call
+      end
+    end
+  end
+
+  # +pipeline+ is an Array of (Arrays or Procs)
+  def run_pipeline(env, pipeline)
+    pids = {} # pid => pipeline index
+    work = pipeline.dup
+    last = work.pop
+    nr = work.size
+    rdr = {} # redirect mapping for Process.spawn
+
+    # we need to make sure pipes are closed in any forked processes
+    # (they are redirected to stdin or stdout, first)
+    pipes = nr.times.map { IO.pipe.each { |io| rdr[io] = :close } }
+
+    # start the first and last commands first, they only have one pipe, each
+    last_pid = pspawn(env, last, rdr.merge(0 => pipes[-1][0]))
+    pids[last_pid] = nr
+    first = work.shift
+    first_pid = pspawn(env, first, rdr.merge(1 => pipes[0][1]))
+    pids[first_pid] = 0
+
+    # start the middle commands, they both have two pipes:
+    work.each_with_index do |cmd, i|
+      pid = pspawn(env, cmd, rdr.merge(0 => pipes[i][0], 1 => pipes[i+1][1]))
+      pids[pid] = i + 1
+    end
+
+    # all pipes handed off to children, close so they see EOF
+    pipes.flatten!.each(&:close).clear
+
+    # wait for children to finish
+    fails = []
+    until pids.empty?
+      pid, status = Process.waitpid2(-1)
+      nr = pids.delete(pid)
+      status.success? or
+        fails << "reaped #{nr} #{pipeline[nr].inspect} #{status.inspect}"
+    end
+    # behave like "set -o pipefail" in bash
+    raise fails.join("\n") if fails[0]
+  end
+end

data/lib/dtas/source/sox.rb

@@ -1,4 +1,4 @@
-# Copyright (C) 2013-2016 all contributors <dtas-all@nongnu.org>
+# Copyright (C) 2013-2019 all contributors <dtas-all@nongnu.org>
 # License: GPL-3.0+ <https://www.gnu.org/licenses/gpl-3.0.txt>
 # frozen_string_literal: true
 # encoding: binary
@@ -50,17 +50,19 @@ def mcache_lookup(infile)
       out =~ /^Sample Rate\s*:\s*(\d+)/n and dst['rate'] = $1.to_i
       out =~ /^Precision\s*:\s*(\d+)-bit/n and dst['bits'] = $1.to_i
 
+      enc = Encoding.default_external # typically Encoding::UTF_8
       if out =~ /\nComments\s*:[ \t]*\n?(.*)\z/mn
         comments = dst['comments'] = {}
         key = nil
         $1.split(/\n/n).each do |line|
           if line.sub!(/^([^=]+)=/ni, '')
-            key = DTAS.dedupe_str($1.upcase)
+            key = DTAS.dedupe_str(DTAS.try_enc($1.upcase, enc))
           end
           (comments[key] ||= ''.b) << "#{line}\n" unless line.empty?
         end
         comments.each do |k,v|
           v.chomp!
+          DTAS.try_enc(v, enc)
           comments[k] = DTAS.dedupe_str(v)
         end
       end

data/lib/dtas/watchable.rb

@@ -1,4 +1,4 @@
-# Copyright (C) 2013-2016 all contributors <dtas-all@nongnu.org>
+# Copyright (C) 2013-2019 all contributors <dtas-all@nongnu.org>
 # License: GPL-3.0+ <https://www.gnu.org/licenses/gpl-3.0.txt>
 # frozen_string_literal: true
 begin
@@ -37,9 +37,13 @@ def watch_files(paths, blk)
       @dir2wd = {}
       Array(paths).each do |path|
         watchdir, watchbase = File.split(File.expand_path(path))
-        wd = @dir2wd[watchdir] ||= add_watch(watchdir, FLAGS)
-        m = @watches[wd] ||= {}
-        m[watchbase] = true
+        begin
+          wd = @dir2wd[watchdir] ||= add_watch(watchdir, FLAGS)
+          m = @watches[wd] ||= {}
+          m[watchbase] = true
+        rescue SystemCallError => e
+          warn "#{watchdir.dump}: #{e.message} (#{e.class})"
+        end
       end
     end
   end

data/test/test_buffer.rb

@@ -1,4 +1,4 @@
-# Copyright (C) 2013-2016 all contributors <dtas-all@nongnu.org>
+# Copyright (C) 2013-2019 all contributors <dtas-all@nongnu.org>
 # License: GPL-3.0+ <https://www.gnu.org/licenses/gpl-3.0.txt>
 # frozen_string_literal: true
 require './test/helper'
@@ -49,14 +49,14 @@ def test_set_buffer_size
     buf = new_buffer
     buf.buffer_size = @@max_size
     assert_equal @@max_size, buf.buffer_size
-  end if @@max_size
+  end if defined?(SleepyPenguin::F_GETPIPE_SZ)
 
   def test_buffer_size
     buf = new_buffer
     assert_operator buf.buffer_size, :>, 128
     buf.buffer_size = @@max_size
     assert_equal @@max_size, buf.buffer_size
-  end if @@max_size
+  end if defined?(SleepyPenguin::F_GETPIPE_SZ)
 
   def test_broadcast_1
     buf = new_buffer
@@ -87,9 +87,9 @@ def test_broadcast_tee
     assert_empty blocked
     assert_equal "HELLO", a[0].read(5)
     assert_equal "HELLO", b[0].read(5)
-    max = '*' * a[0].pipe_size
+    max = '*' * pipe_size(a[0])
     assert_equal max.size, a[1].write(max)
-    assert_equal a[0].nread, a[0].pipe_size
+    assert_equal a[0].nread, pipe_size(a[0])
     a[1].nonblock = true
     assert_equal 5, buf.__broadcast_tee(blocked, [a[1], b[1]], 5)
     assert_equal [a[1]], blocked
@@ -108,10 +108,10 @@ def test_broadcast
     assert_equal "HELLO", a[0].read(5)
     assert_equal "HELLO", b[0].read(5)
 
-    return unless b[1].respond_to?(:pipe_size)
+    return unless defined?(SleepyPenguin::F_GETPIPE_SZ)
 
     b[1].nonblock = true
-    b[1].write('*' * b[1].pipe_size)
+    b[1].write('*' * pipe_size(b[1]))
     buf.wr.write "BYE"
     assert_equal :wait_readable, buf.broadcast([a[1], b[1]])
     assert_equal 8, buf.bytes_xfer
@@ -157,8 +157,8 @@ def test_broadcast_all_full
     a = pipe
     b = pipe
     buf = new_buffer
-    a[1].write('*' * a[1].pipe_size)
-    b[1].write('*' * b[1].pipe_size)
+    a[1].write('*' * pipe_size(a[1]))
+    b[1].write('*' * pipe_size(b[1]))
 
     a[1].nonblock = true
     b[1].nonblock = true
@@ -167,7 +167,7 @@ def test_broadcast_all_full
     buf.wr.write "HELLO"
     assert_equal tmp, buf.broadcast(tmp)
     assert_equal [a[1], b[1]], tmp
-  end if IO.method_defined?(:pipe_size)
+  end if defined?(SleepyPenguin::F_GETPIPE_SZ)
 
   def test_serialize
     buf = new_buffer
@@ -204,4 +204,8 @@ def test_load_size
     assert_equal 4096, buf.buffer_size
     buf.close!
   end
+
+  def pipe_size(io)
+    io.fcntl(SleepyPenguin::F_GETPIPE_SZ)
+  end
 end

data/test/test_encoding.rb

@@ -0,0 +1,20 @@
+# Copyright (C) 2018-2019 all contributors <dtas-all@nongnu.org>
+# License: GPL-3.0+ <https://www.gnu.org/licenses/gpl-3.0.txt>
+# frozen_string_literal: true
+require './test/helper'
+require 'dtas'
+require 'yaml'
+
+class TestEncoding < Testcase
+  def test_encoding
+    data = <<EOD # <20180111114546.77906b35@cumparsita.ch>
+---
+comments:
+  ARTIST: !binary |-
+    RW5yaXF1ZSBSb2Ryw61ndWV6
+EOD
+    hash = YAML.load(data)
+    artist = DTAS.try_enc(hash['comments']['ARTIST'], Encoding::UTF_8)
+    assert_equal 'Enrique Rodríguez', artist
+  end
+end

data/test/test_mlib.rb

@@ -1,10 +1,10 @@
-# Copyright (C) 2013-2016 all contributors <dtas-all@nongnu.org>
+# Copyright (C) 2013-2019 all contributors <dtas-all@nongnu.org>
 # License: GPL-3.0+ <https://www.gnu.org/licenses/gpl-3.0.txt>
 # frozen_string_literal: true
 require_relative 'helper'
 begin
   require 'dtas/mlib'
-  require 'sequel/no_core_ext'
+  require 'sequel'
   require 'sqlite3'
 rescue LoadError => err
   warn "skipping mlib test: #{err.message}"
@@ -13,7 +13,7 @@
 
 class TestMlib < Testcase
   def setup
-    @db = Sequel.sqlite(':memory:')
+    @db = Sequel.sqlite(':memory:', case_sensitive_like: false)
   end
 
   def test_migrate

data/test/test_pipeline.rb

@@ -0,0 +1,47 @@
+# Copyright (C) 2017-2019 all contributors <dtas-all@nongnu.org>
+# License: GPL-3.0+ <https://www.gnu.org/licenses/gpl-3.0.txt>
+# frozen_string_literal: true
+require './test/helper'
+require 'dtas/pipeline'
+
+class TestPipeline < Testcase
+  include DTAS::Pipeline
+  def setup
+    @env = ENV.to_hash
+  end
+
+  def pipeline_result
+    IO.pipe do |rd, wr|
+      begin
+        pid = fork do
+          rd.close
+          $stdout.reopen(wr)
+          yield
+          exit!(0)
+        end
+        wr.close
+        return rd.read
+      ensure
+        _, status = Process.waitpid2(pid)
+        assert_predicate status, :success?
+      end
+    end
+    nil
+  end
+
+  def test_pipeline
+    assert_equal("BYYRU\n", pipeline_result do
+      run_pipeline(@env, [
+        %w(echo hello), # anything which generates something to stdout
+        %w(tr [a-z] [A-Z]), # upcase
+        # this lambda runs inside its own process
+        lambda do
+          $stdin.each_line { |l| $stdout.write("#{l.chomp.reverse}\n") }
+          exit!(0)
+        end,
+        # rot13
+        %w(tr [a-m][n-z][A-M][N-Z] [n-z][a-m][N-Z][A-M])
+      ])
+    end)
+  end
+end

data/test/test_sink_pipe_size.rb

@@ -1,8 +1,8 @@
-# Copyright (C) 2013-2016 all contributors <dtas-all@nongnu.org>
+# Copyright (C) 2013-2019 all contributors <dtas-all@nongnu.org>
 # License: GPL-3.0+ <https://www.gnu.org/licenses/gpl-3.0.txt>
 # frozen_string_literal: true
 begin
-  require 'io/splice'
+  require 'sleepy_penguin'
   require './test/player_integration'
   class TestSinkPipeSizeIntegration < Testcase
     include PlayerIntegration
@@ -14,7 +14,7 @@ def test_sink_pipe_size_integration
       s.req_ok("sink ed default pipe_size=0x10000")
       s.req_ok("sink ed default pipe_size=")
       s.req_ok("sink ed default pipe_size=4096")
-    end if IO.method_defined?(:pipe_size=)
+    end if SleepyPenguin.const_defined?(:F_SETPIPE_SZ)
   end
 rescue LoadError
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: dtas
 version: !ruby/object:Gem::Version
-  version: 0.15.0
+  version: 0.16.0
 platform: ruby
 authors:
 - dtas hackers
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2017-04-07 00:00:00.000000000 Z
+date: 2019-01-02 00:00:00.000000000 Z
 dependencies: []
 description: |-
   Free Software command-line tools for audio playback, mastering, and
@@ -98,6 +98,7 @@ files:
 - lib/dtas/cue_index.rb
 - lib/dtas/disclaimer.rb
 - lib/dtas/edit_client.rb
+- lib/dtas/encoding.rb
 - lib/dtas/fadefx.rb
 - lib/dtas/format.rb
 - lib/dtas/mcache.rb
@@ -108,6 +109,7 @@ files:
 - lib/dtas/parse_time.rb
 - lib/dtas/partstats.rb
 - lib/dtas/pipe.rb
+- lib/dtas/pipeline.rb
 - lib/dtas/player.rb
 - lib/dtas/player/client_handler.rb
 - lib/dtas/process.rb
@@ -164,6 +166,7 @@ files:
 - test/helper.rb
 - test/player_integration.rb
 - test/test_buffer.rb
+- test/test_encoding.rb
 - test/test_env.rb
 - test/test_fadefx.rb
 - test/test_format.rb
@@ -171,6 +174,7 @@ files:
 - test/test_mcache.rb
 - test/test_mlib.rb
 - test/test_parse_freq.rb
+- test/test_pipeline.rb
 - test/test_player.rb
 - test/test_player_client_handler.rb
 - test/test_player_integration.rb
@@ -206,8 +210,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubyforge_project: 
-rubygems_version: 2.6.10
+rubygems_version: 3.0.1
 signing_key: 
 specification_version: 4
 summary: duct tape audio suite for *nix