dtas 0.15.0 → 0.18.1

data/lib/dtas/buffer/read_write.rb

@@ -1,4 +1,4 @@
-# Copyright (C) 2013-2016 all contributors <dtas-all@nongnu.org>
+# Copyright (C) 2013-2020 all contributors <dtas-all@nongnu.org>
 # License: GPL-3.0+ <https://www.gnu.org/licenses/gpl-3.0.txt>
 # frozen_string_literal: true
 require 'io/nonblock'
@@ -6,8 +6,8 @@
 require_relative '../pipe'
 require_relative '../nonblock'
 
-# compatibility code for systems lacking "splice" support via the
-# "io-splice" RubyGem.  Used only by -player
+# compatibility code for non-Linux systems lacking "splice" support.
+# Used only by -player
 module DTAS::Buffer::ReadWrite # :nodoc:
   MAX_AT_ONCE = 512 # min PIPE_BUF value in POSIX
   attr_accessor :buffer_size

data/lib/dtas/buffer/splice.rb

@@ -1,16 +1,19 @@
-# Copyright (C) 2013-2016 all contributors <dtas-all@nongnu.org>
+# Copyright (C) 2013-2020 all contributors <dtas-all@nongnu.org>
 # License: GPL-3.0+ <https://www.gnu.org/licenses/gpl-3.0.txt>
 # frozen_string_literal: true
 require 'io/nonblock'
-require 'io/splice'
+require 'sleepy_penguin'
 require_relative '../../dtas'
 require_relative '../pipe'
+SleepyPenguin.respond_to?(:splice) or
+  raise LoadError, 'sleepy_penguin 3.5+ required for splice', []
 
-# Used by -player on Linux systems with the "io-splice" RubyGem installed
+# Used by -player on Linux systems with the "sleepy_penguin" RubyGem installed
 module DTAS::Buffer::Splice # :nodoc:
   MAX_AT_ONCE = 4096 # page size in Linux
   MAX_AT_ONCE_1 = 65536
-  F_MOVE = IO::Splice::F_MOVE
+  F_MOVE = SleepyPenguin::F_MOVE
+  F_NONBLOCK = SleepyPenguin::F_NONBLOCK
 
   def buffer_size
     @to_io.pipe_size
@@ -24,13 +27,14 @@ def buffer_size=(bytes)
 
   # be sure to only call this with nil when all writers to @wr are done
   def discard(bytes)
-    IO.splice(@to_io, nil, DTAS.null, nil, bytes)
+    SleepyPenguin.splice(@to_io, DTAS.null, bytes)
   end
 
   def broadcast_one(targets, limit = nil)
     # single output is always non-blocking
     limit ||= MAX_AT_ONCE_1
-    s = IO.trysplice(@to_io, nil, targets[0], nil, limit, F_MOVE)
+    s = SleepyPenguin.splice(@to_io, targets[0], limit, F_MOVE|F_NONBLOCK,
+                             exception: false)
     if Symbol === s
       targets # our one and only target blocked on write
     else
@@ -46,7 +50,7 @@ def broadcast_one(targets, limit = nil)
   def __tee_in_full(src, dst, bytes)
     rv = 0
     while bytes > 0
-      s = IO.tee(src, dst, bytes)
+      s = SleepyPenguin.tee(src, dst, bytes)
       bytes -= s
       rv += s
     end
@@ -56,7 +60,7 @@ def __tee_in_full(src, dst, bytes)
   def __splice_in_full(src, dst, bytes, flags)
     rv = 0
     while bytes > 0
-      s = IO.splice(src, nil, dst, nil, bytes, flags)
+      s = SleepyPenguin.splice(src, dst, bytes, flags)
       rv += s
       bytes -= s
     end
@@ -69,7 +73,8 @@ def __broadcast_tee(blocked, targets, chunk_size)
     targets.delete_if do |dst|
       begin
         t = (dst.nonblock? || most_teed == 0) ?
-            IO.trytee(@to_io, dst, chunk_size) :
+            SleepyPenguin.tee(@to_io, dst, chunk_size, F_NONBLOCK,
+                              exception: false) :
             __tee_in_full(@to_io, dst, chunk_size)
         if Integer === t
           if t > most_teed
@@ -117,7 +122,8 @@ def broadcast_inf(targets, limit = nil)
     begin
       targets << last
       if last.nonblock? || most_teed == 0
-        s = IO.trysplice(@to_io, nil, last, nil, bytes, F_MOVE)
+        s = SleepyPenguin.splice(@to_io, last, bytes, F_MOVE|F_NONBLOCK,
+                                 exception: false)
         if Symbol === s
           blocked << last
 

data/lib/dtas/command.rb

@@ -1,4 +1,4 @@
-# Copyright (C) 2013-2016 all contributors <dtas-all@nongnu.org>
+# Copyright (C) 2013-2020 all contributors <dtas-all@nongnu.org>
 # License: GPL-3.0+ <https://www.gnu.org/licenses/gpl-3.0.txt>
 # frozen_string_literal: true
 require_relative 'serialize'

data/lib/dtas/compat_onenine.rb

@@ -1,4 +1,4 @@
-# Copyright (C) 2013-2016 all contributors <dtas-all@nongnu.org>
+# Copyright (C) 2013-2020 all contributors <dtas-all@nongnu.org>
 # License: GPL-3.0+ <https://www.gnu.org/licenses/gpl-3.0.txt>
 
 # Make Ruby 1.9.3 look like Ruby 2.0.0 to us

data/lib/dtas/cue_index.rb

@@ -1,4 +1,4 @@
-# Copyright (C) 2013-2016 all contributors <dtas-all@nongnu.org>
+# Copyright (C) 2013-2020 all contributors <dtas-all@nongnu.org>
 # License: GPL-3.0+ <https://www.gnu.org/licenses/gpl-3.0.txt>
 # frozen_string_literal: true
 require_relative '../dtas'

data/lib/dtas/disclaimer.rb

@@ -1,5 +1,5 @@
 # :enddoc:
-# Copyright (C) 2013-2016 all contributors <dtas-all@nongnu.org>
+# Copyright (C) 2013-2020 all contributors <dtas-all@nongnu.org>
 # License: GPL-3.0+ <https://www.gnu.org/licenses/gpl-3.0.txt>
 # frozen_string_literal: true
 DTAS_PROGNAME = File.basename($0)

data/lib/dtas/edit_client.rb

@@ -1,4 +1,4 @@
-# Copyright (C) 2013-2016 all contributors <dtas-all@nongnu.org>
+# Copyright (C) 2013-2020 all contributors <dtas-all@nongnu.org>
 # License: GPL-3.0+ <https://www.gnu.org/licenses/gpl-3.0.txt>
 # frozen_string_literal: true
 require 'tempfile'

data/lib/dtas/encoding.rb

@@ -0,0 +1,57 @@
+# Copyright (C) 2018-2020 all contributors <dtas-all@nongnu.org>
+# License: GPL-3.0+ <https://www.gnu.org/licenses/gpl-3.0.txt>
+# frozen_string_literal: true
+
+# This module gets included in DTAS
+module DTAS::Encoding # :nodoc:
+  def self.extended(mod)
+    mod.instance_eval { @charlock_holmes = nil}
+  end
+
+private
+
+  def try_enc_harder(str, enc, old) # :nodoc:
+    begin
+      require 'charlock_holmes'
+      @charlock_holmes = CharlockHolmes::EncodingDetector.new
+    rescue LoadError
+      @charlock_holmes = false
+    end if @charlock_holmes.nil?
+
+    case @charlock_holmes
+    when false
+      enc_fallback(str, enc, old)
+    else
+      res = @charlock_holmes.detect(str)
+      if det = res[:ruby_encoding]
+        str.force_encoding(det)
+        warn "charlock_holmes detected #{str.inspect} as #{det}..."
+        str.valid_encoding? or enc_fallback(str, det, old)
+      else
+        enc_fallback(str, enc, old)
+      end
+    end
+    str
+  end
+
+  def enc_fallback(str, enc, old) # :nodoc:
+    str.force_encoding(old)
+    warn "could not detect encoding for #{str.inspect} (not #{enc})"
+  end
+
+public
+
+  def try_enc(str, enc, harder = true) # :nodoc:
+    old = str.encoding
+    return str if old == enc
+    str.force_encoding(enc)
+    unless str.valid_encoding?
+      if harder
+        try_enc_harder(str, enc, old)
+      else
+        enc_fallback(str, enc, old)
+      end
+    end
+    str
+  end
+end

data/lib/dtas/fadefx.rb

@@ -1,4 +1,4 @@
-# Copyright (C) 2013-2016 all contributors <dtas-all@nongnu.org>
+# Copyright (C) 2013-2020 all contributors <dtas-all@nongnu.org>
 # License: GPL-3.0+ <https://www.gnu.org/licenses/gpl-3.0.txt>
 # frozen_string_literal: true
 require_relative '../dtas'

data/lib/dtas/format.rb

@@ -1,4 +1,4 @@
-# Copyright (C) 2013-2016 all contributors <dtas-all@nongnu.org>
+# Copyright (C) 2013-2020 all contributors <dtas-all@nongnu.org>
 # License: GPL-3.0+ <https://www.gnu.org/licenses/gpl-3.0.txt>
 # frozen_string_literal: true
 require_relative '../dtas'

data/lib/dtas/mcache.rb

@@ -1,4 +1,4 @@
-# Copyright (C) 2016 all contributors <dtas-all@nongnu.org>
+# Copyright (C) 2016-2020 all contributors <dtas-all@nongnu.org>
 # License: GPL-3.0+ <https://www.gnu.org/licenses/gpl-3.0.txt>
 # frozen_string_literal: true
 # encoding: binary

data/lib/dtas/mlib.rb

@@ -1,5 +1,5 @@
 # -*- encoding: utf-8 -*-
-# Copyright (C) 2015-2016 all contributors <dtas-all@nongnu.org>
+# Copyright (C) 2015-2020 all contributors <dtas-all@nongnu.org>
 # License: GPL-3.0+ <https://www.gnu.org/licenses/gpl-3.0.txt>
 # frozen_string_literal: true
 #
@@ -10,6 +10,7 @@
 require_relative 'source/ff'
 require_relative 'source/splitfx'
 require 'socket'
+require 'tempfile'
 
 # For the DTAS Music Library, based on what MPD uses.
 class DTAS::Mlib # :nodoc:
@@ -37,16 +38,13 @@ class DTAS::Mlib # :nodoc:
 
   def initialize(db)
     if String === db
+      require 'sequel'
+      opts = { single_threaded: true }
       db = "sqlite://#{db}" unless db.include?('://')
-      require 'sequel/no_core_ext'
-      db = Sequel.connect(db, single_threaded: true)
-    end
-    if db.class.to_s.downcase.include?('sqlite')
-      db.transaction_mode = :immediate
-      db.synchronous = :off
-      db.case_sensitive_like = false
-    else
-      warn 'non-SQLite databases may not work in the future'
+      opts[:transaction_mode] = :immediate
+      opts[:synchronous] = :off
+      opts[:case_sensitive_like] = false # only for 'search'
+      db = Sequel.connect(db, opts)
     end
     @db = db
     @pwd = nil
@@ -65,15 +63,23 @@ def initialize(db)
     ]
   end
 
+  def synchronize
+    @lock.flock(File::LOCK_EX)
+    @db.transaction { yield }
+  ensure
+    @lock.flock(File::LOCK_UN)
+  end
+
   def init_suffixes
-    `sox --help 2>/dev/null` =~ /\nAUDIO FILE FORMATS:\s*([^\n]+)/s
+    `sox --help 2>/dev/null` =~ /\nAUDIO FILE FORMATS:\s*([^\n]+)/
     re = $1.split(/\s+/).map! { |x| Regexp.quote(x) }.join('|')
     @suffixes = Regexp.new("\\.(?:#{re}|yml)\\z", Regexp::IGNORECASE)
   end
 
-  def worker(todo)
+  def worker(todo, lock)
+    @lock = lock
     @work.close
-    @db.tables # reconnect before chdir
+    synchronize { @db.tables } # reconnect before chdir
     @pwd = Dir.pwd.b
     begin
       buf = todo.recv(16384) # 4x bigger than PATH_MAX ought to be enough
@@ -87,7 +93,7 @@ def worker(todo)
   end
 
   def ignore(job)
-    @db.transaction do
+    synchronize do
       node_ensure(job.parent_id, job.path, DM_IGN, job.ctime)
     end
   end
@@ -106,24 +112,16 @@ def worker_work(job)
     end
     return ignore(job) unless found
     tlen = found.duration
-    return ignore(job) if tlen < 0
+    return ignore(job) if tlen.nil? || tlen < 0
     tlen = tlen.round
     tmp = {}
-    found.comments.each do |tag, value|
-      tag_id = @tag_map[tag] or next
-      value.strip!
-
-      # FIXME: this fallback needs testing
-      [ Encoding::UTF_8, Encoding::ISO_8859_1 ].each do |enc|
-        value.force_encoding(enc)
-        if value.valid_encoding?
-          value.encode!(Encoding::UTF_8) if enc != Encoding::UTF_8
-          tmp[tag_id] = value
-          break
-        end
+    if comments = found.comments
+      comments.each do |tag, value|
+        tag_id = @tag_map[tag] or next
+        tmp[tag_id] = value if value.valid_encoding?
       end
     end
-    @db.transaction do
+    synchronize do
       node_id = node_ensure(job.parent_id, path, tlen, job.ctime)[:id]
       vals = @db[:vals]
       comments = @db[:comments]
@@ -154,11 +152,20 @@ def update(path, opts = nil)
     @work and raise 'update already running'
     todo, @work = UNIXSocket.pair(:SOCK_SEQPACKET)
     @db.disconnect
-    jobs.times { |i| fork { worker(todo) } }
+
+    # like a Mutex between processes
+    @lock = Tempfile.new('dtas.mlib.lock')
+    jobs.times do |i|
+      lock = File.open(@lock.path, 'w')
+      fork { worker(todo, lock) }
+      lock.close
+    end
+    @lock.unlink
     todo.close
     scan_dir(path, st)
     @work.close
     Process.waitall
+    @lock.close
   ensure
     @work = nil
   end
@@ -246,7 +253,7 @@ def root_node
   end
 
   def dir_vivify(parts, ctime)
-    @db.transaction do
+    synchronize do
       dir = root_node
       last = parts.pop
       parts.each do |name|
@@ -300,7 +307,7 @@ def scan_dir(path, st, parent_id = nil)
       dir = dir_vivify(@pwd.split(%r{/+}n), st.ctime.to_i)
       dir_id = dir[:id]
 
-      @db.transaction do
+      synchronize do
         @db[:nodes].where(parent_id: dir_id).each do |node|
           File.exist?(node[:name]) or remove_entry(node)
         end

data/lib/dtas/mlib/migrations/0001_initial.rb

@@ -1,4 +1,4 @@
-# Copyright (C) 2015-2016 all contributors <dtas-all@nongnu.org>
+# Copyright (C) 2015-2020 all contributors <dtas-all@nongnu.org>
 # License: GPL-3.0+ <https://www.gnu.org/licenses/gpl-3.0.txt>
 
 Sequel.migration do

data/lib/dtas/nonblock.rb

@@ -1,4 +1,4 @@
-# Copyright (C) 2015-2016 all contributors <dtas-all@nongnu.org>
+# Copyright (C) 2015-2020 all contributors <dtas-all@nongnu.org>
 # License: GPL-3.0+ <https://www.gnu.org/licenses/gpl-3.0.txt>
 
 class DTAS::Nonblock < IO # :nodoc:

data/lib/dtas/parse_freq.rb

@@ -1,4 +1,4 @@
-# Copyright (C) 2015-2016 all contributors <dtas-all@nongnu.org>
+# Copyright (C) 2015-2020 all contributors <dtas-all@nongnu.org>
 # License: GPL-3.0+ <https://www.gnu.org/licenses/gpl-3.0.txt>
 # frozen_string_literal: true
 require_relative '../dtas'

data/lib/dtas/parse_time.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
-# Copyright (C) 2013-2016 all contributors <dtas-all@nongnu.org>
+# Copyright (C) 2013-2020 all contributors <dtas-all@nongnu.org>
 # License: GPL-3.0+ <https://www.gnu.org/licenses/gpl-3.0.txt>
 # frozen_string_literal: true
 require_relative '../dtas'

data/lib/dtas/partstats.rb

@@ -1,5 +1,5 @@
 # -*- encoding: binary -*-
-# Copyright (C) 2013-2016 all contributors <dtas-all@nongnu.org>
+# Copyright (C) 2013-2020 all contributors <dtas-all@nongnu.org>
 # License: GPL-3.0+ <https://www.gnu.org/licenses/gpl-3.0.txt>
 # frozen_string_literal: true
 require_relative '../dtas'

data/lib/dtas/pipe.rb

@@ -1,10 +1,6 @@
-# Copyright (C) 2013-2016 all contributors <dtas-all@nongnu.org>
+# Copyright (C) 2013-2020 all contributors <dtas-all@nongnu.org>
 # License: GPL-3.0+ <https://www.gnu.org/licenses/gpl-3.0.txt>
 # frozen_string_literal: true
-begin
-  require 'io/splice'
-rescue LoadError
-end
 require_relative '../dtas'
 require_relative 'writable_iter'
 require_relative 'nonblock'
@@ -14,18 +10,29 @@ class DTAS::Pipe < DTAS::Nonblock # :nodoc:
   include DTAS::WritableIter
   attr_accessor :sink
 
+  if RUBY_PLATFORM =~ /linux/i && File.readable?('/proc/sys/fs/pipe-max-size')
+    F_SETPIPE_SZ = 1031
+    F_GETPIPE_SZ = 1032
+  end
+
   def self.new
     _, w = rv = pipe
     w.writable_iter_init
     rv
   end
 
-  # create no-op methods for non-Linux
-  unless method_defined?(:pipe_size=)
-    def pipe_size=(_)
-    end
+  def pipe_size=(nr)
+    fcntl(F_SETPIPE_SZ, nr) if defined?(F_SETPIPE_SZ)
+  rescue Errno::EINVAL # old kernel
+  rescue Errno::EPERM
+    # resizes fail if Linux is close to the pipe limit for the user
+    # or if the user does not have permissions to resize
   end
 
+  def pipe_size
+    fcntl(F_GETPIPE_SZ)
+  end if defined?(F_GETPIPE_SZ)
+
   # avoid syscall, we never change IO#nonblock= directly
   def nonblock?
     false

data/lib/dtas/pipeline.rb

@@ -0,0 +1,75 @@
+# Copyright (C) 2017-2020 all contributors <dtas-all@nongnu.org>
+# License: GPL-3.0+ <https://www.gnu.org/licenses/gpl-3.0.txt>
+# frozen_string_literal: true
+require_relative '../dtas'
+require_relative 'spawn_fix'
+
+module DTAS::Pipeline # :nodoc:
+  include DTAS::SpawnFix
+
+  # Process.spawn wrapper which supports running Proc-like objects in
+  # a separate process, not just external commands.
+  # Returns the pid of the spawned process
+  def pspawn(env, cmd, rdr = {})
+    case cmd
+    when Array
+      spawn(env, *cmd, rdr)
+    else # support running Proc-like objects, too:
+      fork do
+        ENV.update(env) if env
+
+        # setup redirects
+        [ $stdin, $stdout, $stderr ].each_with_index do |io, fd|
+          dst = rdr[fd] and io.reopen(dst)
+        end
+
+        # close all other pipes, since we can't rely on FD_CLOEXEC
+        # (as we do not exec, here)
+        rdr.each do |k, v|
+          k.close if v == :close
+        end
+        cmd.call
+      end
+    end
+  end
+
+  # +pipeline+ is an Array of (Arrays or Procs)
+  def run_pipeline(env, pipeline)
+    pids = {} # pid => pipeline index
+    work = pipeline.dup
+    last = work.pop
+    nr = work.size
+    rdr = {} # redirect mapping for Process.spawn
+
+    # we need to make sure pipes are closed in any forked processes
+    # (they are redirected to stdin or stdout, first)
+    pipes = nr.times.map { IO.pipe.each { |io| rdr[io] = :close } }
+
+    # start the first and last commands first, they only have one pipe, each
+    last_pid = pspawn(env, last, rdr.merge(0 => pipes[-1][0]))
+    pids[last_pid] = nr
+    first = work.shift
+    first_pid = pspawn(env, first, rdr.merge(1 => pipes[0][1]))
+    pids[first_pid] = 0
+
+    # start the middle commands, they both have two pipes:
+    work.each_with_index do |cmd, i|
+      pid = pspawn(env, cmd, rdr.merge(0 => pipes[i][0], 1 => pipes[i+1][1]))
+      pids[pid] = i + 1
+    end
+
+    # all pipes handed off to children, close so they see EOF
+    pipes.flatten!.each(&:close).clear
+
+    # wait for children to finish
+    fails = []
+    until pids.empty?
+      pid, status = Process.waitpid2(-1)
+      nr = pids.delete(pid)
+      status.success? or
+        fails << "reaped #{nr} #{pipeline[nr].inspect} #{status.inspect}"
+    end
+    # behave like "set -o pipefail" in bash
+    raise fails.join("\n") if fails[0]
+  end
+end